1. Test Modules
  2. Training Characteristics
    1. Input Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    2. Model Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    3. Composite Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    4. Results
  3. Results

Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase

Test Modules

Using Seed 7558951680825010176

Training Characteristics

Input Learning

In this apply, we use a network to learn this target input, given it's pre-evaluated output:

TrainingTester.java:332 executed in 0.03 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
      try {
        return x.prettyPrint();
      } finally {
        x.freeRef();
      }
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ -0.9793751951731975 ], [ -0.9547362346838266 ], [ 0.6459532369409575 ], [ 0.9716150662353109 ], [ 0.8135802304645023 ], [ 0.22446259266205315 ], [ -0.772969345356455 ], [ 0.7869670723606874 ], ... ],
    	[ [ -0.41685748860755667 ], [ -0.13149352224638 ], [ 0.13014069552467666 ], [ -0.15338715405101788 ], [ 0.21616739519902617 ], [ 0.8300181665729549 ], [ -0.8090351031086032 ], [ -0.5080555160927103 ], ... ],
    	[ [ -0.7928857083877263 ], [ -0.8165069677061334 ], [ -0.8527372758700649 ], [ 0.6212232099150963 ], [ -0.5152394676449892 ], [ -0.3261167026019576 ], [ -0.3314470036403505 ], [ 0.7077997039977317 ], ... ],
    	[ [ 0.049728594061781095 ], [ -0.6312904614448258 ], [ -0.713178706654318 ], [ 0.20449991215112506 ], [ -0.2885063473726639 ], [ -0.44190810546124015 ], [ 0.7658979046776331 ], [ -0.7140791152558215 ], ... ],
    	[ [ 0.9393223813060068 ], [ 0.9463905910213262 ], [ 0.3930191950946851 ], [ 0.9780356710296119 ], [ -0.977630274468841 ], [ -0.8302993228746645 ], [ -0.02937896515371241 ], [ 0.07303407029940057 ], ... ],
    	[ [ -0.5222090565781883 ], [ -0.1747852037729154 ], [ -0.4339502640247199 ], [ -0.1544657946808945 ], [ -0.2145834469531589 ], [ -0.9156873024221812 ], [ 0.18903410938485132 ], [ 0.07845155851669783 ], ... ],
    	[ [ 0.24480787702066809 ], [ 0.08130165010952917 ], [ -0.8146070400068264 ], [ 0.3942853893954561 ], [ -0.26292358392373827 ], [ -0.23725951240167764 ], [ -0.49541149725355904 ], [ -0.1482273173563391 ], ... ],
    	[ [ -0.14762649522252902 ], [ 0.18155717831206508 ], [ -0.20458357985667885 ], [ -0.16420464787509892 ], [ 0.10375784928492915 ], [ 0.5091817864531825 ], [ 0.7302701789956314 ], [ -0.3510416363479689 ], ... ],
    	...
    ]
    [
    	[ [ 0.8325785654129179 ], [ 0.3643670690072889 ], [ -0.004056282458482663 ], [ 0.7175330959686632 ], [ 0.2789739435539107 ], [ -0.18242746197004855 ], [ -0.8046608304902503 ], [ 0.6648090824832684 ], ... ],
    	[ [ 0.46697536601671197 ], [ 0.058538652470407015 ], [ 0.21273084616996019 ], [ -0.34799131807444117 ], [ -0.33042367069653267 ], [ -0.2661619399081192 ], [ -0.40142510639509044 ], [ -0.9087306107243003 ], ... ],
    	[ [ -0.24547174429982044 ], [ 0.45573063338014475 ], [ 0.2368387461187873 ], [ -0.3680188465703589 ], [ -0.8648650647949929 ], [ 0.44149011899409496 ], [ -0.5568500423740361 ], [ -0.510030856062611 ], ... ],
    	[ [ 0.7554451069295298 ], [ -0.01919835387152502 ], [ 0.01876303573913285 ], [ -0.6172027389039059 ], [ -0.030590001233946174 ], [ -0.3303595152143317 ], [ 0.2844071741566614 ], [ 0.03429181664007386 ], ... ],
    	[ [ 0.16311232039237744 ], [ 0.611558284327435 ], [ 0.6283024637081802 ], [ -0.061332444877032044 ], [ 0.6694002980581069 ], [ 0.3593241631656541 ], [ -0.26929451283364014 ], [ 0.6393629313202154 ], ... ],
    	[ [ -0.2867049748265118 ], [ 0.3754812806224226 ], [ -0.6099816922565322 ], [ 0.18363726994453167 ], [ -0.4475333837094886 ], [ -0.5737843715642592 ], [ -0.06595850962889127 ], [ 0.8214770202843333 ], ... ],
    	[ [ 0.7191189571004215 ], [ 0.3563886204697744 ], [ -0.7957970975541828 ], [ -0.17526734582294745 ], [ -0.4831592083494074 ], [ -0.7596745619392768 ], [ -0.4735070486634234 ], [ -0.03405773356117847 ], ... ],
    	[ [ -0.28425920436320884 ], [ 0.9280627411733982 ], [ -0.16255326098321543 ], [ -0.9545114326416864 ], [ 0.20003567492680396 ], [ -0.5221300392750732 ], [ -0.3630732986483036 ], [ -0.6631587060907544 ], ... ],
    	...
    ]
    [
    	[ [ -0.861352346948465 ], [ -0.8560285227925268 ], [ -0.12179792606097761 ], [ -0.6322316657946014 ], [ -0.35026221401253044 ], [ -0.355998738246998 ], [ 0.38547558960082395 ], [ -0.3191618502603386 ], ... ],
    	[ [ 0.25035554753067224 ], [ -0.08420132658045176 ], [ -0.2417943378969123 ], [ 0.15633018116397168 ], [ -0.4600750382097938 ], [ -0.8627316054879579 ], [ -0.8299344318702533 ], [ -0.5740394654446577 ], ... ],
    	[ [ 0.6967233350753874 ], [ 0.7003969320785866 ], [ 0.12084203055476261 ], [ -0.429497096607508 ], [ -0.00794500916912888 ], [ 0.19207430222229327 ], [ -0.3726665741931372 ], [ -0.4751604596340009 ], ... ],
    	[ [ 0.4034557993873149 ], [ 0.40947413345372197 ], [ 0.824326490249486 ], [ -0.2050320481969503 ], [ -0.5791198366306001 ], [ -0.6416276064200759 ], [ 0.6742896141863528 ], [ -0.8774723003633955 ], ... ],
    	[ [ -0.09967002802997138 ], [ -0.6843544505363652 ], [ 0.15776124145780046 ], [ 0.1920303618047654 ], [ 0.7727858769269287 ], [ -0.3194001896162313 ], [ 0.6287032910902701 ], [ 0.5519099617028052 ], ... ],
    	[ [ -0.6002686123495411 ], [ -0.15027282032309963 ], [ -0.38038794070695836 ], [ 0.489716275510486 ], [ -0.05468786990317631 ], [ 0.8477877163869705 ], [ 0.4194667687270095 ], [ -0.40073506981061446 ], ... ],
    	[ [ -0.1525996490759245 ], [ 0.852692561768926 ], [ -0.38576558614966183 ], [ 0.5826171092588548 ], [ 0.6720816486588084 ], [ 0.3132482388693574 ], [ -0.6267202965357168 ], [ -0.6453278681841139 ], ... ],
    	[ [ 0.18842566140988737 ], [ -0.8729414657247996 ], [ -0.2873952137514628 ], [ -0.391070966016438 ], [ -0.6319560904869155 ], [ 0.32967054486499614 ], [ 0.23120429615857252 ], [ -0.6403098494984274 ], ... ],
    	...
    ]
    [
    	[ [ -0.5676806032823238 ], [ -0.7653102637556515 ], [ -0.9735592312258843 ], [ -0.43770043459994357 ], [ -0.7848679699087244 ], [ -0.5471891880039197 ], [ 0.35698524262605136 ], [ -0.3064187728674901 ], ... ],
    	[ [ 0.16296032646580705 ], [ -0.8408411181988933 ], [ 0.696071842057995 ], [ 0.23885798815871984 ], [ 0.7920138183681316 ], [ -0.8731117724589774 ], [ -0.3205277909999438 ], [ -0.19348766571041964 ], ... ],
    	[ [ -0.8527966266560865 ], [ -0.5143578906715243 ], [ 0.45796755201464445 ], [ -0.007052624152861675 ], [ 0.2722394908571444 ], [ -0.5676568698551441 ], [ 0.5487174208527927 ], [ 0.5534953796456559 ], ... ],
    	[ [ 0.9427688231930429 ], [ 0.9859374135775272 ], [ 0.08666690717491687 ], [ -0.6844919115058232 ], [ -0.9178036035788857 ], [ -0.7897025741839996 ], [ -0.011934065687637885 ], [ 0.2517317923958322 ], ... ],
    	[ [ 0.6561872110540334 ], [ 0.8829076092465185 ], [ -0.01131905531682853 ], [ -0.08147460547102625 ], [ -0.8501158852301056 ], [ 0.9980931684731147 ], [ -0.46929302705682874 ], [ 0.8993372540071276 ], ... ],
    	[ [ 0.7628129255808385 ], [ 0.474003246102791 ], [ 0.006491840942871585 ], [ -0.3808290564778163 ], [ -0.9420167415103853 ], [ 0.002249986649791147 ], [ 0.11785758792258205 ], [ 0.782306534795941 ], ... ],
    	[ [ -0.0717166410790353 ], [ -0.4811316707813069 ], [ 0.8650303241484598 ], [ -0.668339118504957 ], [ -0.4334256331708459 ], [ 0.33895101551547313 ], [ -0.3424332916058832 ], [ -0.5996835255715642 ], ... ],
    	[ [ 0.3338973858069856 ], [ -0.09514662542490826 ], [ 0.1636927405320987 ], [ -0.19362440973557216 ], [ -0.7198228613360751 ], [ -0.39108502480123586 ], [ 0.8507008278000725 ], [ 0.15263328304771817 ], ... ],
    	...
    ]
    [
    	[ [ -0.24551699837300323 ], [ 0.3748836953400472 ], [ -0.229855582393655 ], [ 0.6282015809960508 ], [ -0.058237100555295074 ], [ 0.5752095327589861 ], [ 0.5865357384739016 ], [ -0.7405352453279743 ], ... ],
    	[ [ -0.7394559404624785 ], [ -0.6409035514753592 ], [ 0.6889259731106174 ], [ 0.8900296194944153 ], [ 0.5170743297986982 ], [ -0.7911862345326013 ], [ -0.9680696304118821 ], [ 0.4451645734939109 ], ... ],
    	[ [ -0.2853641131548639 ], [ -0.09322248663132982 ], [ 0.5915574729905975 ], [ -0.31128013690850964 ], [ 0.13099099577947326 ], [ 0.7352886830246181 ], [ 0.9168898495639566 ], [ 0.4682973044677772 ], ... ],
    	[ [ -0.06703818897131608 ], [ 0.9257903398736818 ], [ 0.7734164560723273 ], [ -0.5796263504289632 ], [ -0.704386425466816 ], [ -0.5399650318885856 ], [ -0.7617169310039964 ], [ -0.7261551635557415 ], ... ],
    	[ [ 0.10355239779468328 ], [ -0.9671896307349843 ], [ -0.24058705573750383 ], [ 0.18940067906501445 ], [ 0.30823106313096416 ], [ 0.7210556850742524 ], [ 0.77280313993774 ], [ -0.24796504413904774 ], ... ],
    	[ [ 0.8326164547825927 ], [ 0.2109901126813063 ], [ -0.9072032690258975 ], [ -0.4660654520883698 ], [ -0.6900689687047721 ], [ 0.1305886225201951 ], [ 0.03595301811911367 ], [ 0.04641793601462041 ], ... ],
    	[ [ -0.11026666870741475 ], [ -0.5775778987290734 ], [ -0.9693846512148019 ], [ 0.6474364536233057 ], [ 0.8106768791360386 ], [ 0.0691787966883779 ], [ -0.3873917093709822 ], [ -0.8235620571135673 ], ... ],
    	[ [ -0.6093369509140532 ], [ -0.873836215221 ], [ 0.1771846796466221 ], [ -4.5725723890033265E-4 ], [ -0.24878603745574002 ], [ 0.6770168133110295 ], [ 0.8291886407238591 ], [ 0.22108541186124064 ], ... ],
    	...
    ]

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:480 executed in 3.42 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 4635130838967
BACKPROP_AGG_SIZE = 3
THREADS = 64
SINGLE_THREADED = false
Initialized CoreSettings = {
"backpropAggregationSize" : 3,
"jvmThreads" : 64,
"singleThreaded" : false
}
Reset training subject: 4635217546533
Constructing line search parameters: GD
th(0)=2918.5930231719344;dx=-12.828475319424383
New Minimum: 2918.5930231719344 > 2891.0167578927
WOLFE (weak): th(2.154434690031884)=2891.0167578927; dx=-12.771149456575792 evalInputDelta=27.576265279234576
New Minimum: 2891.0167578927 > 2863.563432763508
WOLFE (weak): th(4.308869380063768)=2863.563432763508; dx=-12.714346566392024 evalInputDelta=55.02959040842643
New Minimum: 2863.563432763508 > 2754.9573889509306
WOLFE (weak): th(12.926608140191302)=2754.9573889509306; dx=-12.492220886463922 evalInputDelta=163.63563422100378
New Minimum: 2754.9573889509306 > 2288.5834379102153
WOLFE (weak): th(51.70643256076521)=2288.5834379102153; dx=-11.583568291494657 evalInputDelta=630.0095852617192
New Minimum: 2288.5834379102153 > 271.0408217982928
END: th(258.53216280382605)=271.0408217982928; dx=-8.374174205531089 evalInputDelta=2647.5522013736418
Fitness changed from 2918.5930231719344 to 271.0408217982928
Iteration 1 complete. Error: 271.0408217982928 Total: 0.7673; Orientation: 0.0162; Line Search: 0.6162
th(0)=271.0408217982928;dx=-5.4842841349622695
New Minimum: 271.0408217982928 > 0.0
WOLFE (weak): th(556.9906600335335)=0.0; dx=-5.254902762081752 evalInputDelta=271.0408217982928
WOLFE (weak): th(1113.981320067067)=0.0; dx=-5.254902762081752 evalInputDelta=271.0408217982928
WOLFE (weak): th(3341.9439602012008)=0.0; dx=-5.254902762081752 evalInputDelta=271.0408217982928
WOLFE (weak): th(13367.775840804803)=0.0; dx=-5.254902762081752 evalInputDelta=271.0408217982928
WOLFE (weak): th(66838.87920402401)=0.0; dx=-5.254902762081752 evalInputDelta=271.0408217982928
WOLFE (weak): th(401033.27522414405)=0.0; dx=-5.254902762081752 evalInputDelta=271.0408217982928
WOLFE (weak): th(2807232.9265690083)=0.0; dx=-5.254902762081752 evalInputDelta=271.0408217982928
WOLFE (weak): th(2.2457863412552066E7)=0.0; dx=-5.254902762081752 evalInputDelta=271.0408217982928
MAX ALPHA (2.021207707129686E8): th(556.9906600335335)=0.0
Fitness changed from 271.0408217982928 to 0.0
Iteration 2 complete. Error: 0.0 Total: 0.6730; Orientation: 0.0052; Line Search: 0.6522
th(0)=0.0;dx=-5.040138840565945
Armijo: th(1.0E8)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(5.0E7)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(1.6666666666666666E7)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(4166666.6666666665)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(833333.3333333333)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(138888.88888888888)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(19841.26984126984)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(2480.15873015873)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(275.5731922398589)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(27.55731922398589)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(2.505210838544172)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(0.20876756987868098)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(0.016059043836821614)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(0.0011470745597729724)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(7.647163731819817E-5)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(4.7794773323873855E-6)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(2.811457254345521E-7)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(1.5619206968586227E-8)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(8.22063524662433E-10)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(4.110317623312165E-11)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(1.9572941063391265E-12)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(8.896791392450575E-14)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Armijo: th(3.868170170630685E-15)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
MIN ALPHA (1.6117375710961187E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 1.9725; Orientation: 0.0041; Line Search: 1.9543
Iteration 3 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 3
Final threshold in iteration 3: 0.0 (> 0.0) after 3.414s (< 30.000s)

Returns

    0.0

Training Converged

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:452 executed in 3.72 seconds (0.019 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 4638549402995
Reset training subject: 4638559338422
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=2918.5930231719344}, derivative=-12.828475319424383}
New Minimum: 2918.5930231719344 > 2918.5930231706557
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=2918.5930231706557}, derivative=-12.828475319421711}, evalInputDelta = -1.278749550692737E-9
New Minimum: 2918.5930231706557 > 2918.593023162949
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=2918.593023162949}, derivative=-12.828475319405673}, evalInputDelta = -8.985352906165645E-9
New Minimum: 2918.593023162949 > 2918.593023109069
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=2918.593023109069}, derivative=-12.828475319293403}, evalInputDelta = -6.286563802859746E-8
New Minimum: 2918.593023109069 > 2918.5930227319272
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=2918.5930227319272}, derivative=-12.828475318507516}, evalInputDelta = -4.4000717025483027E-7
New Minimum: 2918.5930227319272 > 2918.5930200918224
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=2918.5930200918224}, derivative=-12.828475313006312}, evalInputDelta = -3.0801120374235325E-6
New Minimum: 2918.5930200918224 > 2918.593001611112
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=2918.593001611112}, derivative=-12.828475274497887}, evalInputDelta = -2.15608224607422E-5
New Minimum: 2918.593001611112 > 2918.5928722462086
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=2918.5928722462086}, derivative=-12.828475004938912}, evalInputDelta = -1.509257258476282E-4
New Minimum: 2918.5928722462086 > 2918.5919666919253
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=2918.5919666919253}, derivative=-12.828473118026416}, evalInputDelta = -0.001056480009083316
New Minimum: 2918.5919666919253 > 2918.5856278156434
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=2918.5856278156434}, derivative=-12.828459909654885}, evalInputDelta = -0.007395356290999189
New Minimum: 2918.5856278156434 > 2918.541255864432
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=2918.541255864432}, derivative=-12.828367451835371}, evalInputDelta = -0.05176730750235947
New Minimum: 2918.541255864432 > 2918.230661160053
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=2918.230661160053}, derivative=-12.827720285375264}, evalInputDelta = -0.362362011881487
New Minimum: 2918.230661160053 > 2916.0569368563497
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=2916.0569368563497}, derivative=-12.82319199489358}, evalInputDelta = -2.5360863155847255
New Minimum: 2916.0569368563497 > 2900.862317280842
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=2900.862317280842}, derivative=-12.791585546940023}, evalInputDelta = -17.730705891092384
New Minimum: 2900.862317280842 > 2795.536819391657
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=2795.536819391657}, derivative=-12.574735171224543}, evalInputDelta = -123.05620378027743
New Minimum: 2795.536819391657 > 2104.6643694850104
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=2104.6643694850104}, derivative=-11.244443145129997}, evalInputDelta = -813.928653686924
New Minimum: 2104.6643694850104 > 0.0
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-8.019675178742542}, evalInputDelta = -2918.5930231719344
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-8.019675178742542}, evalInputDelta = -2918.5930231719344
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-8.019675178742542}, evalInputDelta = -2918.5930231719344
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-8.019675178742542}, evalInputDelta = -2918.5930231719344
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-8.019675178742542}, evalInputDelta = -2918.5930231719344
F(7979226.6297612) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-8.019675178742542}, evalInputDelta = -2918.5930231719344
F(5.58545864083284E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-8.019675178742542}, evalInputDelta = -2918.5930231719344
F(3.909821048582988E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-8.019675178742542}, evalInputDelta = -2918.5930231719344
F(2.7368747340080914E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-8.019675178742542}, evalInputDelta = -2918.5930231719344
F(1.915812313805664E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-8.019675178742542}, evalInputDelta = -2918.5930231719344
0.0 <= 2918.5930231719344
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-8.019675178742542}, evalInputDelta = -2918.5930231719344
Right bracket at 1.0E10
Converged to right
Fitness changed from 2918.5930231719344 to 0.0
Iteration 1 complete. Error: 0.0 Total: 1.6926; Orientation: 0.0045; Line Search: 1.6593
F(0.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.040138840565945}
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.040138840565945}, evalInputDelta = 0.0
0.0 <= 0.0
F(5.0E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.040138840565945}, evalInputDelta = 0.0
Right bracket at 5.0E9
F(2.5E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.040138840565945}, evalInputDelta = 0.0
Right bracket at 2.5E9
F(1.25E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.040138840565945}, evalInputDelta = 0.0
Right bracket at 1.25E9
F(6.25E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.040138840565945}, evalInputDelta = 0.0
Right bracket at 6.25E8
F(3.125E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.040138840565945}, evalInputDelta = 0.0
Right bracket at 3.125E8
F(1.5625E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.040138840565945}, evalInputDelta = 0.0
Right bracket at 1.5625E8
F(7.8125E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.040138840565945}, evalInputDelta = 0.0
Right bracket at 7.8125E7
F(3.90625E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.040138840565945}, evalInputDelta = 0.0
Right bracket at 3.90625E7
F(1.953125E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.040138840565945}, evalInputDelta = 0.0
Right bracket at 1.953125E7
F(9765625.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.040138840565945}, evalInputDelta = 0.0
Right bracket at 9765625.0
F(4882812.5) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.040138840565945}, evalInputDelta = 0.0
Right bracket at 4882812.5
F(2441406.25) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.040138840565945}, evalInputDelta = 0.0
Loops = 12
Fitness changed from 0.0 to 0.0
Static Iteration Total: 2.0288; Orientation: 0.0027; Line Search: 2.0045
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 3.721s (< 30.000s)

Returns

    0.0

Training Converged

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:509 executed in 8.44 seconds (0.015 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 4642275941029
Reset training subject: 4642284479662
Adding measurement 41f4ba29 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 2918.5930231719344 < 2918.5930231719344. Total: 1
th(0)=2918.5930231719344;dx=-12.828475319424383
Adding measurement 70c3915b to history. Total: 1
New Minimum: 2918.5930231719344 > 2891.0167578927
WOLFE (weak): th(2.154434690031884)=2891.0167578927; dx=-12.771149456575792 evalInputDelta=27.576265279234576
Adding measurement 16f14dfe to history. Total: 2
New Minimum: 2891.0167578927 > 2863.563432763508
WOLFE (weak): th(4.308869380063768)=2863.563432763508; dx=-12.714346566392024 evalInputDelta=55.02959040842643
Adding measurement 4eda3112 to history. Total: 3
New Minimum: 2863.563432763508 > 2754.9573889509306
WOLFE (weak): th(12.926608140191302)=2754.9573889509306; dx=-12.492220886463922 evalInputDelta=163.63563422100378
Adding measurement 6bead17c to history. Total: 4
New Minimum: 2754.9573889509306 > 2288.5834379102153
WOLFE (weak): th(51.70643256076521)=2288.5834379102153; dx=-11.583568291494657 evalInputDelta=630.0095852617192
Adding measurement 15d34f2b to history. Total: 5
New Minimum: 2288.5834379102153 > 271.0408217982928
END: th(258.53216280382605)=271.0408217982928; dx=-8.374174205531089 evalInputDelta=2647.5522013736418
Fitness changed from 2918.5930231719344 to 271.0408217982928
Iteration 1 complete. Error: 271.0408217982928 Total: 0.7008; Orientation: 0.0204; Line Search: 0.5818
Non-optimal measurement 271.0408217982928 < 271.0408217982928. Total: 6
Rejected: LBFGS Orientation magnitude: 1.189e+03, gradient 2.342e+00, dot -0.990; [40a6f257-bba0-453e-9ec6-b10e8fc7c8b0 = 1.000/1.000e+00, 3ac1fdb3-24e1-4b6a-88a2-3a8275f190c4 = 1.000/1.000e+00, 1dc4d423-cdd4-43aa-8f8b-639a0fcf57df = 1.000/1.000e+00, 5ed82054-9fa1-4ca3-85ca-c09ad117cd9a = 1.000/1.000e+00, b0755c40-48a7-441e-9d0f-99bbdedd4f45 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 271.0408217982928, 2288.5834379102153, 2754.9573889509306, 2863.563432763508, 2891.0167578927, 2918.5930231719344
Rejected: LBFGS Orientation magnitude: 1.199e+03, gradient 2.342e+00, dot -0.990; [b0755c40-48a7-441e-9d0f-99bbdedd4f45 = 1.000/1.000e+00, 1dc4d423-cdd4-43aa-8f8b-639a0fcf57df = 1.000/1.000e+00, 40a6f257-bba0-453e-9ec6-b10e8fc7c8b0 = 1.000/1.000e+00, 3ac1fdb3-24e1-4b6a-88a2-3a8275f190c4 = 1.000/1.000e+00, 5ed82054-9fa1-4ca3-85ca-c09ad117cd9a = 1.000/1.000e+00]
Orientation rejected. Popping history element from 271.0408217982928, 2288.5834379102153, 2754.9573889509306, 2863.563432763508, 2891.0167578927
Rejected: LBFGS Orientation magnitude: 1.225e+03, gradient 2.342e+00, dot -0.990; [40a6f257-bba0-453e-9ec6-b10e8fc7c8b0 = 1.000/1.000e+00, 1dc4d423-cdd4-43aa-8f8b-639a0fcf57df = 1.000/1.000e+00, 5ed82054-9fa1-4ca3-85ca-c09ad117cd9a = 1.000/1.000e+00, b0755c40-48a7-441e-9d0f-99bbdedd4f45 = 1.000/1.000e+00, 3ac1fdb3-24e1-4b6a-88a2-3a8275f190c4 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 271.0408217982928, 2288.5834379102153, 2754.9573889509306, 2863.563432763508
LBFGS Accumulation History: 3 points
Removed measurement 15d34f2b to history. Total: 5
Removed measurement 6bead17c to history. Total: 4
Removed measurement 4eda3112 to history. Total: 3
Adding measurement 16fc352d to history. Total: 3
th(0)=271.0408217982928;dx=-5.4842841349622695
Adding measurement 34e2da2 to history. Total: 4
New Minimum: 271.0408217982928 > 0.0
WOLFE (weak): th(556.9906600335335)=0.0; dx=-5.254902762081752 evalInputDelta=271.0408217982928
Non-optimal measurement 0.0 < 0.0. Total: 5
WOLFE (weak): th(1113.981320067067)=0.0; dx=-5.254902762081752 evalInputDelta=271.0408217982928
Non-optimal measurement 0.0 < 0.0. Total: 5
WOLFE (weak): th(3341.9439602012008)=0.0; dx=-5.254902762081752 evalInputDelta=271.0408217982928
Non-optimal measurement 0.0 < 0.0. Total: 5
WOLFE (weak): th(13367.775840804803)=0.0; dx=-5.254902762081752 evalInputDelta=271.0408217982928
Non-optimal measurement 0.0 < 0.0. Total: 5
WOLFE (weak): th(66838.87920402401)=0.0; dx=-5.254902762081752 evalInputDelta=271.0408217982928
Non-optimal measurement 0.0 < 0.0. Total: 5
WOLFE (weak): th(401033.27522414405)=0.0; dx=-5.254902762081752 evalInputDelta=271.0408217982928
Non-optimal measurement 0.0 < 0.0. Total: 5
WOLFE (weak): th(2807232.9265690083)=0.0; dx=-5.254902762081752 evalInputDelta=271.0408217982928
Non-optimal measurement 0.0 < 0.0. Total: 5
WOLFE (weak): th(2.2457863412552066E7)=0.0; dx=-5.254902762081752 evalInputDelta=271.0408217982928
Non-optimal measurement 0.0 < 0.0. Total: 5
MAX ALPHA (2.021207707129686E8): th(556.9906600335335)=0.0
Fitness changed from 271.0408217982928 to 0.0
Iteration 2 complete. Error: 0.0 Total: 3.7511; Orientation: 3.2968; Line Search: 0.4436
Non-optimal measurement 0.0 < 0.0. Total: 5
Rejected: LBFGS Orientation magnitude: 1.153e+03, gradient 2.245e+00, dot -0.986; [b0755c40-48a7-441e-9d0f-99bbdedd4f45 = 1.000/1.000e+00, 1dc4d423-cdd4-43aa-8f8b-639a0fcf57df = 1.000/1.000e+00, 40a6f257-bba0-453e-9ec6-b10e8fc7c8b0 = 1.000/1.000e+00, 5ed82054-9fa1-4ca3-85ca-c09ad117cd9a = 1.000/1.000e+00, 3ac1fdb3-24e1-4b6a-88a2-3a8275f190c4 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 271.0408217982928, 2863.563432763508, 2891.0167578927, 2918.5930231719344
Rejected: LBFGS Orientation magnitude: 1.163e+03, gradient 2.245e+00, dot -0.986; [b0755c40-48a7-441e-9d0f-99bbdedd4f45 = 1.000/1.000e+00, 5ed82054-9fa1-4ca3-85ca-c09ad117cd9a = 1.000/1.000e+00, 40a6f257-bba0-453e-9ec6-b10e8fc7c8b0 = 1.000/1.000e+00, 1dc4d423-cdd4-43aa-8f8b-639a0fcf57df = 1.000/1.000e+00, 3ac1fdb3-24e1-4b6a-88a2-3a8275f190c4 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.0, 271.0408217982928, 2863.563432763508, 2891.0167578927
LBFGS Accumulation History: 3 points
Removed measurement 34e2da2 to history. Total: 4
Removed measurement 16fc352d to history. Total: 3
Adding measurement 27d5381f to history. Total: 3
th(0)=0.0;dx=-5.040138840565945
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1.0E8)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(5.0E7)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1.6666666666666666E7)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(4166666.6666666665)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(833333.3333333333)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(138888.88888888888)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(19841.26984126984)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(2480.15873015873)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(275.5731922398589)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(27.55731922398589)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(2.505210838544172)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.20876756987868098)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.016059043836821614)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(0.0011470745597729724)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(7.647163731819817E-5)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(4.7794773323873855E-6)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(2.811457254345521E-7)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1.5619206968586227E-8)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(8.22063524662433E-10)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(4.110317623312165E-11)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(1.9572941063391265E-12)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(8.896791392450575E-14)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
Armijo: th(3.868170170630685E-15)=0.0; dx=-5.040138840565945 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 4
MIN ALPHA (1.6117375710961187E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 3.9875; Orientation: 1.8924; Line Search: 2.0854
Iteration 3 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 3
Final threshold in iteration 3: 0.0 (> 0.0) after 8.440s (< 30.000s)

Returns

    0.0

Training Converged

TrainingTester.java:432 executed in 0.10 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging
Plotting range=[1.0, 1.4330347054375179], [2.0, 3.433034705437518]; valueStats=DoubleSummaryStatistics{count=2, sum=542.081644, min=271.040822, average=271.040822, max=271.040822}
Plotting 2 points for GD
Only 1 points for CjGD
Plotting 2 points for LBFGS

Returns

Result

TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging
Plotting range=[0.0, 1.4330347054375179], [3.751, 3.433034705437518]; valueStats=DoubleSummaryStatistics{count=2, sum=542.081644, min=271.040822, average=271.040822, max=271.040822}
Plotting 2 points for GD
Only 1 points for CjGD
Plotting 2 points for LBFGS

Returns

Result

Model Learning

In this apply, attempt to train a network to emulate a randomized network given an example input/output. The target state is:

TrainingTester.java:370 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> temp_18_0042 = network_target.state();
    assert temp_18_0042 != null;
    String temp_18_0041 = temp_18_0042.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b).orElse("");
    temp_18_0042.freeRef();
    return temp_18_0041;

Returns

    [-0.5399029657838563, -0.9406055442523473]

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:480 executed in 0.01 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 4651176730857
Final threshold in iteration 0: 0.0 (> 0.0) after 0.010s (< 30.000s)

Returns

    0.0

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    [0.7078415864060545, -0.7428102541043857]

To produce the following output:

TrainingTester.java:633 executed in 0.01 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ -1.3689603443259628 ], [ -1.4216900514472999 ], [ -0.5409885277448611 ], [ -0.1749473489784944 ], [ -0.26510627905502454 ], [ -0.7277773953628263 ], [ -1.2620642294703086 ], [ -1.1525222098911174 ], ... ],
    	[ [ -0.9355944915200426 ], [ -0.0797719053955166 ], [ -0.21948475264695988 ], [ -0.40559299360443896 ], [ -1.101346465620911 ], [ -0.47460362411850293 ], [ -0.643110193144592 ], [ -1.0549259220885094 ], ... ],
    	[ [ -0.90646377830979 ], [ -1.2998327527881444 ], [ -0.4630297373075508 ], [ -1.2373573647830898 ], [ -0.4442191251991293 ], [ -0.8846131968514399 ], [ -1.3518353269731165 ], [ -0.9867646835188933 ], ... ],
    	[ [ -1.3251770911731233 ], [ -0.1991465766883842 ], [ -0.7120129737922014 ], [ -0.6843495365482949 ], [ -0.9141096252723152 ], [ -0.44753701038302346 ], [ -0.6737816714543589 ], [ -0.46246072137203276 ], ... ],
    	[ [ -1.3842040910272755 ], [ -1.135409510016915 ], [ -0.1372268231662036 ], [ -1.422950410229014 ], [ -0.20011999346966758 ], [ -0.6682854558380413 ], [ -0.9546932423684801 ], [ -1.3981594090305474 ], ... ],
    	[ [ -0.6003727469304897 ], [ -1.436386880428243 ], [ -0.44563345594945625 ], [ -0.4870630271171048 ], [ -0.9036421990745669 ], [ -0.31762797705698775 ], [ -0.16032814860702282 ], [ -0.5172056902295308 ], ... ],
    	[ [ -0.4351933936826032 ], [ -1.3732902025555038 ], [ -0.9386734920123943 ], [ -0.2101343474359174 ], [ -0.23610513307580272 ], [ -1.1427964970695736 ], [ -0.7572255579419158 ], [ -1.3898841612956563 ], ... ],
    	[ [ -0.11607414812513683 ], [ -1.3766895310667786 ], [ -0.2535206150078309 ], [ -0.6968912918788961 ], [ -0.4644712519011917 ], [ -0.8826317408133765 ], [ -0.7392674049152749 ], [ -0.6333195797626558 ], ... ],
    	...
    ]

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:452 executed in 0.01 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 4651202855069
Final threshold in iteration 0: 0.0 (> 0.0) after 0.007s (< 30.000s)

Returns

    0.0

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    [0.7078415864060545, -0.7428102541043857]

To produce the following output:

TrainingTester.java:633 executed in 0.01 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ -1.3689603443259628 ], [ -1.4216900514472999 ], [ -0.5409885277448611 ], [ -0.1749473489784944 ], [ -0.26510627905502454 ], [ -0.7277773953628263 ], [ -1.2620642294703086 ], [ -1.1525222098911174 ], ... ],
    	[ [ -0.9355944915200426 ], [ -0.0797719053955166 ], [ -0.21948475264695988 ], [ -0.40559299360443896 ], [ -1.101346465620911 ], [ -0.47460362411850293 ], [ -0.643110193144592 ], [ -1.0549259220885094 ], ... ],
    	[ [ -0.90646377830979 ], [ -1.2998327527881444 ], [ -0.4630297373075508 ], [ -1.2373573647830898 ], [ -0.4442191251991293 ], [ -0.8846131968514399 ], [ -1.3518353269731165 ], [ -0.9867646835188933 ], ... ],
    	[ [ -1.3251770911731233 ], [ -0.1991465766883842 ], [ -0.7120129737922014 ], [ -0.6843495365482949 ], [ -0.9141096252723152 ], [ -0.44753701038302346 ], [ -0.6737816714543589 ], [ -0.46246072137203276 ], ... ],
    	[ [ -1.3842040910272755 ], [ -1.135409510016915 ], [ -0.1372268231662036 ], [ -1.422950410229014 ], [ -0.20011999346966758 ], [ -0.6682854558380413 ], [ -0.9546932423684801 ], [ -1.3981594090305474 ], ... ],
    	[ [ -0.6003727469304897 ], [ -1.436386880428243 ], [ -0.44563345594945625 ], [ -0.4870630271171048 ], [ -0.9036421990745669 ], [ -0.31762797705698775 ], [ -0.16032814860702282 ], [ -0.5172056902295308 ], ... ],
    	[ [ -0.4351933936826032 ], [ -1.3732902025555038 ], [ -0.9386734920123943 ], [ -0.2101343474359174 ], [ -0.23610513307580272 ], [ -1.1427964970695736 ], [ -0.7572255579419158 ], [ -1.3898841612956563 ], ... ],
    	[ [ -0.11607414812513683 ], [ -1.3766895310667786 ], [ -0.2535206150078309 ], [ -0.6968912918788961 ], [ -0.4644712519011917 ], [ -0.8826317408133765 ], [ -0.7392674049152749 ], [ -0.6333195797626558 ], ... ],
    	...
    ]

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:509 executed in 0.01 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 4651222212182
Final threshold in iteration 0: 0.0 (> 0.0) after 0.007s (< 30.000s)

Returns

    0.0

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    [0.7078415864060545, -0.7428102541043857]

To produce the following output:

TrainingTester.java:633 executed in 0.01 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ -1.3689603443259628 ], [ -1.4216900514472999 ], [ -0.5409885277448611 ], [ -0.1749473489784944 ], [ -0.26510627905502454 ], [ -0.7277773953628263 ], [ -1.2620642294703086 ], [ -1.1525222098911174 ], ... ],
    	[ [ -0.9355944915200426 ], [ -0.0797719053955166 ], [ -0.21948475264695988 ], [ -0.40559299360443896 ], [ -1.101346465620911 ], [ -0.47460362411850293 ], [ -0.643110193144592 ], [ -1.0549259220885094 ], ... ],
    	[ [ -0.90646377830979 ], [ -1.2998327527881444 ], [ -0.4630297373075508 ], [ -1.2373573647830898 ], [ -0.4442191251991293 ], [ -0.8846131968514399 ], [ -1.3518353269731165 ], [ -0.9867646835188933 ], ... ],
    	[ [ -1.3251770911731233 ], [ -0.1991465766883842 ], [ -0.7120129737922014 ], [ -0.6843495365482949 ], [ -0.9141096252723152 ], [ -0.44753701038302346 ], [ -0.6737816714543589 ], [ -0.46246072137203276 ], ... ],
    	[ [ -1.3842040910272755 ], [ -1.135409510016915 ], [ -0.1372268231662036 ], [ -1.422950410229014 ], [ -0.20011999346966758 ], [ -0.6682854558380413 ], [ -0.9546932423684801 ], [ -1.3981594090305474 ], ... ],
    	[ [ -0.6003727469304897 ], [ -1.436386880428243 ], [ -0.44563345594945625 ], [ -0.4870630271171048 ], [ -0.9036421990745669 ], [ -0.31762797705698775 ], [ -0.16032814860702282 ], [ -0.5172056902295308 ], ... ],
    	[ [ -0.4351933936826032 ], [ -1.3732902025555038 ], [ -0.9386734920123943 ], [ -0.2101343474359174 ], [ -0.23610513307580272 ], [ -1.1427964970695736 ], [ -0.7572255579419158 ], [ -1.3898841612956563 ], ... ],
    	[ [ -0.11607414812513683 ], [ -1.3766895310667786 ], [ -0.2535206150078309 ], [ -0.6968912918788961 ], [ -0.4644712519011917 ], [ -0.8826317408133765 ], [ -0.7392674049152749 ], [ -0.6333195797626558 ], ... ],
    	...
    ]

TrainingTester.java:432 executed in 0.00 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging
No Data

TrainingTester.java:435 executed in 0.00 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging
No Data

Composite Learning

In this apply, attempt to train a network to emulate a randomized network given an example input/output. The target state is:

TrainingTester.java:279 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> temp_18_0037 = temp_18_0035.state();
    assert temp_18_0037 != null;
    String temp_18_0036 = temp_18_0037.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b).orElse("");
    temp_18_0037.freeRef();
    return temp_18_0036;

Returns

    [0.8984273707363721, -0.1765881259448745]

We simultaneously regress this target input:

TrainingTester.java:287 executed in 0.02 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(x -> {
      RefStream<Tensor> temp_18_0006 = RefArrays.stream(RefUtil.addRef(x));
      if (null != x)
        RefUtil.freeRef(x);
      return temp_18_0006;
    }).map(x -> {
      String temp_18_0007 = x.prettyPrint();
      x.freeRef();
      return temp_18_0007;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ 0.6360782606558182 ], [ -0.1535591989878753 ], [ 0.0836282153180572 ], [ -0.2957708794118834 ], [ -0.8234685158202133 ], [ 0.11119558447102462 ], [ 0.9574564968921508 ], [ 0.44011180173835207 ], ... ],
    	[ [ 0.6720444561095744 ], [ 0.5821521705862098 ], [ -0.31747829369924485 ], [ 0.6045510733384938 ], [ 0.41995145696455083 ], [ -0.24047681086380845 ], [ -0.43261414226475203 ], [ -0.9511709323821831 ], ... ],
    	[ [ -0.4116454352213976 ], [ -0.4864883474690749 ], [ 0.41724918236759456 ], [ 0.788998991697293 ], [ 0.5334377012821558 ], [ -0.11672870574195127 ], [ 0.7974037990495955 ], [ -0.6417771538742358 ], ... ],
    	[ [ 0.6956433751450066 ], [ -0.63424980216483 ], [ 0.4470763719768993 ], [ 0.16634124424890406 ], [ 0.32684766173855895 ], [ -0.7454628745245973 ], [ 0.2923711640516691 ], [ 0.9412448520815624 ], ... ],
    	[ [ -0.21710144461828684 ], [ -0.8891126786183965 ], [ -0.5837302558233521 ], [ 0.397730696770749 ], [ 0.017088063483313043 ], [ -0.8891456920744447 ], [ 0.2919003786824972 ], [ -0.14094395704105245 ], ... ],
    	[ [ 0.9601401592424912 ], [ -0.1853059279710041 ], [ 0.08749916181852035 ], [ -0.17281709720623284 ], [ 0.4680367136985857 ], [ -0.982694229248817 ], [ -0.6356779916833546 ], [ -0.25964967646575987 ], ... ],
    	[ [ -0.5122339287999278 ], [ -0.590263861999943 ], [ -0.621182872042515 ], [ -0.4997089059496638 ], [ -0.9706821128213667 ], [ 0.8731840480207078 ], [ -0.1939527071607936 ], [ 0.3337382361802008 ], ... ],
    	[ [ 0.4884302650703043 ], [ 0.7321612321743474 ], [ -0.9642787467890979 ], [ -0.7152672828860245 ], [ -0.8865774998445362 ], [ 0.7727370115682577 ], [ -0.20022555478774606 ], [ -0.9113071344757853 ], ... ],
    	...
    ]
    [
    	[ [ -0.46585784027725774 ], [ 0.9684282209534911 ], [ -0.10133834722657098 ], [ 0.7723795038436791 ], [ -0.5075860902566092 ], [ 0.15222353364692354 ], [ -0.24464184617985274 ], [ 0.814624317410025 ], ... ],
    	[ [ 0.6626412461536508 ], [ -0.14286030012615059 ], [ 0.724411747709568 ], [ 0.695243874930149 ], [ -0.1766085605936527 ], [ 0.8192790931699778 ], [ 0.7975246449981943 ], [ 0.4292652660115799 ], ... ],
    	[ [ 0.589794997259419 ], [ -0.8134201591062782 ], [ 0.8603728617225002 ], [ 0.7679274853127604 ], [ -0.24165633704410405 ], [ -0.4400309756926535 ], [ -0.6935560939747549 ], [ -0.13906477108728033 ], ... ],
    	[ [ -0.8844651692169496 ], [ -0.4516136682177616 ], [ -0.06584803152811891 ], [ 0.08356956895861467 ], [ -0.639861943888498 ], [ 0.14668757636618723 ], [ 0.13005232375530995 ], [ 0.7598622800584285 ], ... ],
    	[ [ -0.3767022520735199 ], [ 0.6868473692572967 ], [ 0.4475236143548007 ], [ -0.9398742265811257 ], [ 0.9555009847337206 ], [ 0.6257584604526452 ], [ 0.6510021861051609 ], [ -0.24427172325250335 ], ... ],
    	[ [ -0.21033381239131432 ], [ -0.009524394469572872 ], [ 0.26584743892738283 ], [ 0.13105139693696355 ], [ 0.405219096818374 ], [ -0.8170569650110109 ], [ -0.5485991746219039 ], [ -0.23531346823447485 ], ... ],
    	[ [ 0.8476707629736213 ], [ 0.6365723802715968 ], [ -0.08154454047660487 ], [ -0.8028786984266176 ], [ -0.3764267837152335 ], [ 0.6827822834015977 ], [ -0.7954788029550155 ], [ -0.16886038906264145 ], ... ],
    	[ [ -0.8463450803960542 ], [ 0.5914202525051437 ], [ 0.612840531725833 ], [ -0.17085114435324078 ], [ 0.4692512710830512 ], [ -0.014601876788782553 ], [ -0.6997391708362211 ], [ -0.19661555118805984 ], ... ],
    	...
    ]
    [
    	[ [ -0.7068864361404434 ], [ -0.021222397683634764 ], [ 0.6246301802514431 ], [ 0.25963624195831336 ], [ -0.9538453304378993 ], [ 0.1271549582380327 ], [ 0.307420196380797 ], [ -0.7153824487706912 ], ... ],
    	[ [ -0.08569072782891252 ], [ -0.0597887805306172 ], [ 0.30009495226640515 ], [ 0.3519211986249666 ], [ -0.2897667510490116 ], [ -0.5464285778087192 ], [ -0.09467063028207456 ], [ -0.18815504780687742 ], ... ],
    	[ [ 0.2842970100735629 ], [ 0.331935358044549 ], [ 0.4684788673912159 ], [ 0.06684697168517006 ], [ -0.7268511737487129 ], [ 0.14055250514782558 ], [ -0.012692073190834652 ], [ 0.3832808175251521 ], ... ],
    	[ [ -0.22865792102842564 ], [ 0.5780196874270804 ], [ -0.5373909682544538 ], [ -0.5099742419223421 ], [ -0.7748982533991997 ], [ 0.8690851093720262 ], [ -0.234732643331365 ], [ -0.1124316235271956 ], ... ],
    	[ [ -0.5436106521799544 ], [ -0.8784657960358686 ], [ -0.6070897159491468 ], [ -0.1439278606723675 ], [ -0.9586708095102681 ], [ -0.42891064675771773 ], [ 0.031386825169445354 ], [ -0.5739466325229712 ], ... ],
    	[ [ 0.45232115269476814 ], [ 0.04939339139280796 ], [ 0.2401949757763917 ], [ -0.5755068185506445 ], [ 0.9104247277039161 ], [ 0.9604435818303934 ], [ 0.7877248609331262 ], [ -0.7024792815325156 ], ... ],
    	[ [ 0.7838848205296693 ], [ -0.11303484779308604 ], [ -0.8731549620007211 ], [ -0.009879778939218875 ], [ 0.12356826424515832 ], [ -0.8404673544276013 ], [ 0.5081974628698827 ], [ 0.44493819215217933 ], ... ],
    	[ [ -0.38094942873824467 ], [ 0.8266500797964498 ], [ -0.1405970097910414 ], [ -0.8523170547240675 ], [ 0.33055911002685745 ], [ -0.7203116988763931 ], [ 0.8859499030438902 ], [ -0.7753923580794124 ], ... ],
    	...
    ]
    [
    	[ [ 0.7024756747660059 ], [ -0.8070429487905839 ], [ -0.7419496065618876 ], [ 0.3454297704613498 ], [ -0.541078495685356 ], [ 0.6498325754535412 ], [ -0.27663149526584085 ], [ 0.6963353872305378 ], ... ],
    	[ [ 0.6107756502141517 ], [ 0.5148078544998507 ], [ -0.16270185945217763 ], [ 0.21785168147784661 ], [ -0.2963653449952137 ], [ -0.4857088488155543 ], [ -0.9788283271580178 ], [ -0.10252260180554273 ], ... ],
    	[ [ 0.45299118744297195 ], [ 0.21856374097096976 ], [ -0.21804946329362962 ], [ -0.6045389172261266 ], [ -0.05816435686022903 ], [ 0.8129340185714509 ], [ 0.9288376616570286 ], [ 0.24703568069921977 ], ... ],
    	[ [ 0.9618549994508876 ], [ 0.6372198285354558 ], [ 0.23991379163744941 ], [ -0.41842254318055927 ], [ -0.9791931259734332 ], [ 0.5439970188763525 ], [ 0.04758812477275587 ], [ -0.30664242732287383 ], ... ],
    	[ [ 0.0805945427303234 ], [ 0.23612289736193026 ], [ 0.17341937500744486 ], [ 0.5466426094834149 ], [ 0.8143616841693919 ], [ -0.9374943595917011 ], [ -0.4345871069935925 ], [ -0.9862470374613315 ], ... ],
    	[ [ -0.29851810826083214 ], [ -0.866206151303305 ], [ -0.09137815742812139 ], [ 0.17742670581479247 ], [ 0.48226265124443457 ], [ 0.291112178484894 ], [ 0.5698594899479255 ], [ -0.43179060120685375 ], ... ],
    	[ [ -0.05570280212299994 ], [ -0.8373708976051182 ], [ -0.21423100027028252 ], [ -0.6393315012018057 ], [ -0.7792475403326373 ], [ 0.5290234051771996 ], [ -0.03787196974950935 ], [ 0.558885590604272 ], ... ],
    	[ [ 0.8447183415327564 ], [ -0.9065886823395461 ], [ 0.06356643453507815 ], [ -0.29422860252370286 ], [ 0.7461928426029489 ], [ -0.06564720864612128 ], [ 0.45623645682071734 ], [ -0.36367352703399947 ], ... ],
    	...
    ]
    [
    	[ [ -0.4991910648861493 ], [ 0.4412209633842752 ], [ 0.32952758497772194 ], [ -0.018180065052495475 ], [ -0.15208449156024595 ], [ 0.8040245329887317 ], [ 0.0728598370040281 ], [ 0.6276963527884227 ], ... ],
    	[ [ -0.7203103654804146 ], [ -0.830290469904164 ], [ 0.10989806069753638 ], [ 0.9932848327604658 ], [ -0.6447906829674677 ], [ 0.20452984088727 ], [ -0.5087369485243738 ], [ 0.3000706604394203 ], ... ],
    	[ [ 0.4295063305108844 ], [ 0.8432572431719212 ], [ -0.16928826249522966 ], [ 0.316798671258282 ], [ 0.9631724749878912 ], [ 0.7145731851559716 ], [ -0.566890472948459 ], [ 0.9154481178266682 ], ... ],
    	[ [ -0.6060588041863357 ], [ 0.5130603597678123 ], [ -0.900829914627149 ], [ 0.6661955672488746 ], [ -0.9491531269641316 ], [ 0.8372204176994442 ], [ 0.0963279679915372 ], [ -0.42999053450076063 ], ... ],
    	[ [ 0.8214498859211383 ], [ -0.14476262242679794 ], [ 0.034061252958093924 ], [ -0.3526340177488747 ], [ -0.12523302413738358 ], [ -0.2769177405653318 ], [ 0.24663027717019048 ], [ -0.9501216592361152 ], ... ],
    	[ [ 0.052919793316009045 ], [ 0.577510261684572 ], [ 0.6848067945325884 ], [ 0.48512584880094445 ], [ 0.5307230950480435 ], [ -0.619444671594692 ], [ -0.9308881866685099 ], [ -0.9605643716927574 ], ... ],
    	[ [ 0.7579482586015809 ], [ -0.22230448489730126 ], [ 0.747743191122155 ], [ -0.6381287619850542 ], [ -0.5624075542174172 ], [ 0.38107823784442396 ], [ 0.9983694587717817 ], [ 0.37168289128565024 ], ... ],
    	[ [ -0.44104020965049107 ], [ -0.3626203499208791 ], [ 0.01585031379966373 ], [ 0.6144160387303861 ], [ 0.42535035977165725 ], [ -0.08451899468566837 ], [ 0.27768417340386176 ], [ -0.8370442256264292 ], ... ],
    	...
    ]

Which produces the following output:

TrainingTester.java:308 executed in 0.02 seconds (0.000 gc):

    return RefStream.of(RefUtil.addRef(output_target)).map(x -> {
      String temp_18_0008 = x.prettyPrint();
      x.freeRef();
      return temp_18_0008;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ 0.394881993358697 ], [ -0.3145499133439347 ], [ -0.10145424833729717 ], [ -0.4423167794752775 ], [ -0.9164147794974113 ], [ -0.07668696935107767 ], [ 0.683616997152398 ], [ 0.21882036292096063 ], ... ],
    	[ [ 0.4271950077756057 ], [ 0.34643331804336597 ], [ -0.4618193146189568 ], [ 0.36655710535048014 ], [ 0.20070775737269564 ], [ -0.39263907485231375 ], [ -0.5652605123231664 ], [ -1.0311461258458627 ], ... ],
    	[ [ -0.5464216519864644 ], [ -0.613662572855398 ], [ 0.1982799599115445 ], [ 0.5322701636793732 ], [ 0.30266690546970704 ], [ -0.2814603901340754 ], [ 0.539821272650448 ], [ -0.7531782868987763 ], ... ],
    	[ [ 0.4483969225568296 ], [ -0.7464155080938869 ], [ 0.22507752344868742 ], [ -0.027142599229314945 ], [ 0.11706075942223015 ], [ -0.8463323762855864 ], [ 0.08608613025319906 ], [ 0.6690520117299091 ], ... ],
    	[ [ -0.37163800601635005 ], [ -0.9753912920843735 ], [ -0.7010273649035187 ], [ 0.18074401821601482 ], [ -0.16123574199858534 ], [ -0.9754209522768899 ], [ 0.08566316379179284 ], [ -0.30321603469044744 ], ... ],
    	[ [ 0.6860280728617585 ], [ -0.3430720435937272 ], [ -0.0979764840506249 ], [ -0.3318517362061623 ], [ 0.24390886815143803 ], [ -1.0594675185666949 ], [ -0.7476986326479282 ], [ -0.4098645020845568 ], ... ],
    	[ [ -0.6367931077985557 ], [ -0.70689733552218 ], [ -0.7346758204204995 ], [ -0.62554028445078 ], [ -1.0486755043878015 ], [ 0.6079043224873121 ], [ -0.35084054668654785 ], [ 0.12325144010069772 ], ... ],
    	[ [ 0.26223099289030827 ], [ 0.4812055648326269 ], [ -1.0429225450795676 ], [ -0.8192038302819143 ], [ -0.9731136180842275 ], [ 0.5176599556290769 ], [ -0.35647624468706063 ], [ -0.9953313987052518 ], ... ],
    	...
    ]
    [
    	[ [ -0.595127560522096 ], [ 0.693474294353273 ], [ -0.2676332707984122 ], [ 0.5173387609040658 ], [ -0.6326173624364748 ], [ -0.039826336846269295 ], [ -0.39638105658033157 ], [ 0.555292657683726 ], ... ],
    	[ [ 0.4187469065784232 ], [ -0.304937729769821 ], [ 0.47424321588037277 ], [ 0.4480380006291864 ], [ -0.33525809068856516 ], [ 0.5594746356311079 ], [ 0.5399298439583118 ], [ 0.2090755383463586 ], ... ],
    	[ [ 0.35329984271637105 ], [ -0.9073870607946894 ], [ 0.5963944020653996 ], [ 0.5133389456008629 ], [ -0.3936987934571915 ], [ -0.5719239984789857 ], [ -0.7996979039128018 ], [ -0.3015277225948752 ], ... ],
    	[ [ -0.971215842432359 ], [ -0.5823302064703664 ], [ -0.2357477997788481 ], [ -0.10150693783181437 ], [ -0.7514576098268819 ], [ -0.04479999239051011 ], [ -0.05974555865523595 ], [ 0.5060929444497643 ], ... ],
    	[ [ -0.5150277398257571 ], [ 0.4404943501141527 ], [ 0.22547933824234728 ], [ -1.0209968561550364 ], [ 0.6818601115054566 ], [ 0.3856104023956356 ], [ 0.40829005646121563 ], [ -0.39604852801186385 ], ... ],
    	[ [ -0.3655577799885604 ], [ -0.1851451026260289 ], [ 0.06225648962765232 ], [ -0.05884796396346968 ], [ 0.18747180178182465 ], [ -0.910654466761557 ], [ -0.6694646399885755 ], [ -0.38800018650963053 ], ... ],
    	[ [ 0.5849824888836106 ], [ 0.39532592394593025 ], [ -0.2498499730431763 ], [ -0.8979163239925412 ], [ -0.5147802515129007 ], [ 0.4368421657169995 ], [ -0.8912680553602658 ], [ -0.3282969213119443 ], ... ],
    	[ [ -0.936967711260765 ], [ 0.35476001651356315 ], [ 0.3740045816542459 ], [ -0.330085470353457 ], [ 0.24500005974897182 ], [ -0.18970685171603688 ], [ -0.8052529494005097 ], [ -0.3532329186446457 ], ... ],
    	...
    ]
    [
    	[ [ -0.8116742481757375 ], [ -0.19565490889650416 ], [ 0.3845967245810158 ], [ 0.05667618026560553 ], [ -1.0335488782593625 ], [ -0.06234863113898559 ], [ 0.0996065928007841 ], [ -0.819307298464874 ], ... ],
    	[ [ -0.25357502124469045 ], [ -0.2303040028365309 ], [ 0.09302539299108897 ], [ 0.13958751124214686 ], [ -0.4369225062166589 ], [ -0.6675145164007772 ], [ -0.2616428113951539 ], [ -0.34563177083678376 ], ... ],
    	[ [ 0.07883208932372854 ], [ 0.12163168503752592 ], [ 0.24430611113096917 ], [ -0.11653097693207845 ], [ -0.8296111148925766 ], [ -0.05031190829450316 ], [ -0.18799103189090968 ], [ 0.16776185119793513 ], ... ],
    	[ [ -0.38202066073248797 ], [ 0.34272058206409706 ], [ -0.6593948806111967 ], [ -0.6347629432584388 ], [ -0.8727779263345246 ], [ 0.6042217238143675 ], [ -0.38747835751907134 ], [ -0.27759977385803447 ], ... ],
    	[ [ -0.6649828148871955 ], [ -0.9658258413592141 ], [ -0.7220141432461575 ], [ -0.30589685538446054 ], [ -1.037884220734894 ], [ -0.5619331905922478 ], [ -0.1483893431321275 ], [ -0.6922374899454823 ], ... ],
    	[ [ 0.22978957799913113 ], [ -0.1322117511840815 ], [ 0.0392096146059957 ], [ -0.6936392037761844 ], [ 0.6413623684195324 ], [ 0.6863006760196295 ], [ 0.5311254497269484 ], [ -0.8077147398489082 ], ... ],
    	[ [ 0.5276754523237492 ], [ -0.2781417270492028 ], [ -0.9610544427005993 ], [ -0.1854643897606935 ], [ -0.06557101519263965 ], [ -0.931687001373019 ], [ 0.27999038443620927 ], [ 0.22315652417060272 ], ... ],
    	[ [ -0.5188435195896985 ], [ 0.5660969317656621 ], [ -0.3029043277848358 ], [ -0.942333096454387 ], [ 0.1203952261495102 ], [ -0.8237358716770418 ], [ 0.6193735160509916 ], [ -0.8732218435032366 ], ... ],
    	...
    ]
    [
    	[ [ 0.45453524754140706 ], [ -0.9016576004981274 ], [ -0.8431759601871569 ], [ 0.13375543450478455 ], [ -0.6627078561854605 ], [ 0.4072392462386957 ], [ -0.42512143289943505 ], [ 0.44901864515535117 ], ... ],
    	[ [ 0.3721494355868239 ], [ 0.2859293412078592 ], [ -0.3227639297464132 ], [ 0.019135787455764858 ], [ -0.4428508636263022 ], [ -0.612962249929623 ], [ -1.0559942863157339 ], [ -0.2686972375260803 ], ... ],
    	[ [ 0.2303915555562619 ], [ 0.01977552119397935 ], [ -0.3724897319422472 ], [ -0.7197224358561567 ], [ -0.22884457614938214 ], [ 0.5537740469424273 ], [ 0.6579050522585698 ], [ 0.04535549114379547 ], ... ],
    	[ [ 0.6875687322414211 ], [ 0.39590760918731693 ], [ 0.03895699107935299 ], [ -0.5525103912714104 ], [ -1.0563220315563153 ], [ 0.3121536854126315 ], [ -0.13383365212701304 ], [ -0.4520840756807831 ], ... ],
    	[ [ -0.10417978282396985 ], [ 0.03555114790265876 ], [ -0.020783412822190872 ], [ 0.31453055642577943 ], [ 0.5550567007918762 ], [ -1.0188587185130256 ], [ -0.5670330778370543 ], [ -1.0626594587077949 ], ... ],
    	[ [ -0.4447849650668496 ], [ -0.9548114409759749 ], [ -0.2586847636657559 ], [ -0.017183117141274712 ], [ 0.2566898398170149 ], [ 0.08495502316064629 ], [ 0.33538923729821024 ], [ -0.5645206204958255 ], ... ],
    	[ [ -0.22663304799888973 ], [ -0.9289050598113967 ], [ -0.3690591202479274 ], [ -0.7509810455985505 ], [ -0.8766854447587109 ], [ 0.2987009810264794 ], [ -0.2106133401515336 ], [ 0.32552998576416614 ], ... ],
    	[ [ 0.5823299526511886 ], [ -0.991092212158545 ], [ -0.1194783012984385 ], [ -0.440931155705682 ], [ 0.4938119476971925 ], [ -0.23556737500499128 ], [ 0.23330719439064096 ], [ -0.5033223766444536 ], ... ],
    	...
    ]
    [
    	[ [ -0.6250750418656273 ], [ 0.219816864102229 ], [ 0.11946847581176667 ], [ -0.1929215939898042 ], [ -0.31322499582712426 ], [ 0.5457695212357312 ], [ -0.11112885415306491 ], [ 0.3873514579116384 ], ... ],
    	[ [ -0.8237346737175987 ], [ -0.9225438097683395 ], [ -0.07785270022336065 ], [ 0.7158061547444279 ], [ -0.7558857239186462 ], [ 0.0071670812406040285 ], [ -0.6336513250040727 ], [ 0.09300356854884062 ], ... ],
    	[ [ 0.20929211729064662 ], [ 0.5810172618924763 ], [ -0.3286813345149925 ], [ 0.10803247132648008 ], [ 0.6887523883241408 ], [ 0.46540398199351984 ], [ -0.685898043051457 ], [ 0.6458755195996997 ], ... ],
    	[ [ -0.7210879439016339 ], [ 0.2843593441103783 ], [ -0.9859183776240146 ], [ 0.4219402059347579 ], [ -1.0293332742294652 ], [ 0.5755936126556445 ], [ -0.09004444293386031 ], [ -0.5629033912979202 ], ... ],
    	[ [ 0.5614249352549466 ], [ -0.30664682819268474 ], [ -0.1459865640057477 ], [ -0.4934041793431992 ], [ -0.28910090254998866 ], [ -0.42537860351124235 ], [ 0.04499126551712243 ], [ -1.0302034301320568 ], ... ],
    	[ [ -0.12904353517606026 ], [ 0.34226290003366977 ], [ 0.4386610419294419 ], [ 0.2592622148696088 ], [ 0.30022802892820893 ], [ -0.7331141735623491 ], [ -1.012923551943013 ], [ -1.0395854488278338 ], ... ],
    	[ [ 0.5043733351847557 ], [ -0.3763125598140604 ], [ 0.4952048232410279 ], [ -0.749900471766363 ], [ -0.6818704661627023 ], [ 0.16578299332654117 ], [ 0.7203743219229523 ], [ 0.1573419568205851 ], ... ],
    	[ [ -0.5728307218901836 ], [ -0.5023761734997931 ], [ -0.1623477701924962 ], [ 0.37542006026992325 ], [ 0.20555827942654548 ], [ -0.2525223041176009 ], [ 0.07289093586145987 ], [ -0.9286115687644899 ], ... ],
    	...
    ]

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:480 executed in 1.25 seconds (0.036 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 4651285836954
Reset training subject: 4651295654847
Constructing line search parameters: GD
th(0)=39532.25946121215;dx=-2.046971593083253E30
New Minimum: 39532.25946121215 > 0.0
Armijo: th(2.154434690031884)=0.0; dx=-2.1514318474072386E26 evalInputDelta=39532.25946121215
Armijo: th(1.077217345015942)=0.0; dx=-1.0757159339384774E26 evalInputDelta=39532.25946121215
Armijo: th(0.3590724483386473)=0.0; dx=-3.5857199162596985E25 evalInputDelta=39532.25946121215
Armijo: th(0.08976811208466183)=0.0; dx=-8.964301325877938E24 evalInputDelta=39532.25946121215
Armijo: th(0.017953622416932366)=0.0; dx=-1.7928619027528622E24 evalInputDelta=39532.25946121215
Armijo: th(0.002992270402822061)=0.0; dx=-2.9881202293513785E23 evalInputDelta=39532.25946121215
Armijo: th(4.2746720040315154E-4)=0.0; dx=-4.268918639495665E22 evalInputDelta=39532.25946121215
Armijo: th(5.343340005039394E-5)=0.0; dx=-5.337939399513524E21 evalInputDelta=39532.25946121215
Armijo: th(5.9370444500437714E-6)=0.0; dx=-5.949239080286881E20 evalInputDelta=39532.25946121215
Armijo: th(5.937044450043771E-7)=0.0; dx=-6.133466523664372E19 evalInputDelta=39532.25946121215
Armijo: th(5.397313136403428E-8)=0.0; dx=-7.43676192431603E18 evalInputDelta=39532.25946121215
Armijo: th(4.4977609470028565E-9)=0.0; dx=-2.4961207873526538E18 evalInputDelta=39532.25946121215
Armijo: th(3.4598161130791205E-10)=0.0; dx=-2.08152153110397901E18 evalInputDelta=39532.25946121215
Armijo: th(2.4712972236279432E-11)=0.0; dx=-2.04943944579902234E18 evalInputDelta=39532.25946121215
Armijo: th(1.6475314824186289E-12)=0.0; dx=-2.04713611659764045E18 evalInputDelta=39532.25946121215
Armijo: th(1.029707176511643E-13)=0.0; dx=-2.04698187580290534E18 evalInputDelta=39532.25946121215
Armijo: th(6.057101038303783E-15)=0.0; dx=-2.0469721979491177E18 evalInputDelta=39532.25946121215
MIN ALPHA (3.3650561323909904E-16): th(2.154434690031884)=0.0
Fitness changed from 39532.25946121215 to 0.0
Iteration 1 complete. Error: 0.0 Total: 0.8194; Orientation: 0.0053; Line Search: 0.7852
th(0)=0.0;dx=-7.22948938210888E27
Armijo: th(2.154434690031884E-15)=0.0; dx=-7.229489382109016E27 evalInputDelta=0.0
Armijo: th(1.077217345015942E-15)=0.0; dx=-7.229489382108948E27 evalInputDelta=0.0
MIN ALPHA (3.5907244833864734E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.4313; Orientation: 0.0029; Line Search: 0.4204
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 1.251s (< 30.000s)

Returns

    0.0

Training Converged

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:452 executed in 4.59 seconds (0.022 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 4652540651800
Reset training subject: 4652556017627
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=39532.25946121215}, derivative=-2.046971593083253E30}
New Minimum: 39532.25946121215 > 0.0
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.05695765502744858E18}, evalInputDelta = -39532.25946121215
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.11687402669260262E18}, evalInputDelta = -39532.25946121215
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.5362886283486812E18}, evalInputDelta = -39532.25946121215
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.4721908399412337E18}, evalInputDelta = -39532.25946121215
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.6023506321089077E19}, evalInputDelta = -39532.25946121215
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-1.698827146891241E20}, evalInputDelta = -39532.25946121215
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-1.1768971732653686E21}, evalInputDelta = -39532.25946121215
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-8.225998383299084E21}, evalInputDelta = -39532.25946121215
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.756970685353506E22}, evalInputDelta = -39532.25946121215
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-4.029756661451873E23}, evalInputDelta = -39532.25946121215
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.820817381186751E24}, evalInputDelta = -39532.25946121215
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-1.97457093864777E25}, evalInputDelta = -39532.25946121215
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-1.3821995342351432E26}, evalInputDelta = -39532.25946121215
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-9.675396616827705E26}, evalInputDelta = -39532.25946121215
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.772777619497564E27}, evalInputDelta = -39532.25946121215
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-4.740944332420112E28}, evalInputDelta = -39532.25946121215
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.3186610325712614E29}, evalInputDelta = -39532.25946121215
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.3230627227876015E30}, evalInputDelta = -39532.25946121215
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-1.6261439059500916E31}, evalInputDelta = -39532.25946121215
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-1.1383007341649411E32}, evalInputDelta = -39532.25946121215
F(7979226.6297612) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-7.968105139154469E32}, evalInputDelta = -39532.25946121215
F(5.58545864083284E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.577673597408112E33}, evalInputDelta = -39532.25946121215
F(3.909821048582988E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.9043715181856778E34}, evalInputDelta = -39532.25946121215
F(2.7368747340080914E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.7330600627299758E35}, evalInputDelta = -39532.25946121215
F(1.915812313805664E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-1.9131420439109836E36}, evalInputDelta = -39532.25946121215
0.0 <= 39532.25946121215
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-9.98606194419235E35}, evalInputDelta = -39532.25946121215
Right bracket at 1.0E10
F(1.4777992907955253E-6) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-1.4962092418275967E20}, evalInputDelta = -39532.25946121215
Right bracket at 1.4777992907955253E-6
F(2.9555985815910507E-6) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.971948767724361E20}, evalInputDelta = -39532.25946121215
Right bracket at 2.9555985815910507E-6
F(5.911197163182101E-6) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.923427819517886E20}, evalInputDelta = -39532.25946121215
Right bracket at 5.911197163182101E-6
F(1.1822394326364203E-5) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-1.182638592310494E21}, evalInputDelta = -39532.25946121215
Right bracket at 1.1822394326364203E-5
F(2.3644788652728405E-5) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.363230213027906E21}, evalInputDelta = -39532.25946121215
Right bracket at 2.3644788652728405E-5
F(4.728957730545681E-5) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-4.724413454462726E21}, evalInputDelta = -39532.25946121215
Right bracket at 4.728957730545681E-5
F(9.457915461091362E-5) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-9.446779937332365E21}, evalInputDelta = -39532.25946121215
Right bracket at 9.457915461091362E-5
F(1.8915830922182724E-4) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-1.8891512903071663E22}, evalInputDelta = -39532.25946121215
Right bracket at 1.8915830922182724E-4
F(3.783166184436545E-4) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.778097883455023E22}, evalInputDelta = -39532.25946121215
Right bracket at 3.783166184436545E-4
F(7.56633236887309E-4) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-7.555991069750736E22}, evalInputDelta = -39532.25946121215
Right bracket at 7.56633236887309E-4
F(0.001513266473774618) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-1.511177744234217E23}, evalInputDelta = -39532.25946121215
Loops = 12
Fitness changed from 39532.25946121215 to 0.0
Iteration 1 complete. Error: 0.0 Total: 2.9683; Orientation: 0.0057; Line Search: 2.9184
F(0.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-1.764240643300326E7}
F(0.001513266473774618) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-1.7849994651518248E7}, evalInputDelta = 0.0
F(0.010592865316422326) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-1.9095523962608162E7}, evalInputDelta = 0.0
F(0.07415005721495628) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.7814229140237555E7}, evalInputDelta = 0.0
F(0.519050400504694) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-8.884516538364331E7}, evalInputDelta = 0.0
F(3.633352803532858) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.1606171908748317E8}, evalInputDelta = 0.0
F(25.433469624730005) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-3.506577595014362E9}, evalInputDelta = 0.0
F(178.03428737311003) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.444018872650254E10}, evalInputDelta = 0.0
F(1246.2400116117701) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-1.7097546664691965E11}, evalInputDelta = 0.0
F(8723.680081282391) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-1.1967224120898403E12}, evalInputDelta = 0.0
F(61065.76056897674) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-8.376951030190286E12}, evalInputDelta = 0.0
F(427460.32398283714) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-5.8638551356893336E13}, evalInputDelta = 0.0
F(2992222.26787986) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-4.1046975364381494E14}, evalInputDelta = 0.0
F(2.094555587515902E7) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.873288169652266E15}, evalInputDelta = 0.0
F(1.4661889112611315E8) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-2.0113017081711428E16}, evalInputDelta = 0.0
F(1.026332237882792E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-1.40791119466125424E17}, evalInputDelta = 0.0
F(7.184325665179544E9) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-9.8553783615702413E17}, evalInputDelta = 0.0
F(5.029027965625681E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-6.8987648529933128E18}, evalInputDelta = 0.0
0.0 <= 0.0
F(1.0E10) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-1.37178892227657293E18}, evalInputDelta = 0.0
Left bracket at 1.0E10
Converged to left
Fitness changed from 0.0 to 0.0
Static Iteration Total: 1.6201; Orientation: 0.0474; Line Search: 1.5064
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 4.588s (< 30.000s)

Returns

    0.0

Training Converged

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:509 executed in 1.95 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 4657132504617
Reset training subject: 4657141406098
Adding measurement 7e51348d to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 39532.25946121215 < 39532.25946121215. Total: 1
th(0)=39532.25946121215;dx=-2.0469715930832533E30
Adding measurement 446880c4 to history. Total: 1
New Minimum: 39532.25946121215 > 0.0
Armijo: th(2.154434690031884)=0.0; dx=-2.1514318474072386E26 evalInputDelta=39532.25946121215
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.077217345015942)=0.0; dx=-1.0757159339384774E26 evalInputDelta=39532.25946121215
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.3590724483386473)=0.0; dx=-3.5857199162596985E25 evalInputDelta=39532.25946121215
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.08976811208466183)=0.0; dx=-8.964301325877937E24 evalInputDelta=39532.25946121215
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.017953622416932366)=0.0; dx=-1.7928619027528622E24 evalInputDelta=39532.25946121215
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(0.002992270402822061)=0.0; dx=-2.9881202293513785E23 evalInputDelta=39532.25946121215
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(4.2746720040315154E-4)=0.0; dx=-4.268918639495665E22 evalInputDelta=39532.25946121215
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(5.343340005039394E-5)=0.0; dx=-5.337939399513525E21 evalInputDelta=39532.25946121215
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(5.9370444500437714E-6)=0.0; dx=-5.949239080286881E20 evalInputDelta=39532.25946121215
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(5.937044450043771E-7)=0.0; dx=-6.133466523664371E19 evalInputDelta=39532.25946121215
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(5.397313136403428E-8)=0.0; dx=-7.4367619243160279E18 evalInputDelta=39532.25946121215
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(4.4977609470028565E-9)=0.0; dx=-2.4961207873526533E18 evalInputDelta=39532.25946121215
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(3.4598161130791205E-10)=0.0; dx=-2.08152153110397926E18 evalInputDelta=39532.25946121215
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(2.4712972236279432E-11)=0.0; dx=-2.04943944579902234E18 evalInputDelta=39532.25946121215
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.6475314824186289E-12)=0.0; dx=-2.04713611659764045E18 evalInputDelta=39532.25946121215
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.029707176511643E-13)=0.0; dx=-2.04698187580290534E18 evalInputDelta=39532.25946121215
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(6.057101038303783E-15)=0.0; dx=-2.0469721979491177E18 evalInputDelta=39532.25946121215
Non-optimal measurement 0.0 < 0.0. Total: 2
MIN ALPHA (3.3650561323909904E-16): th(2.154434690031884)=0.0
Fitness changed from 39532.25946121215 to 0.0
Iteration 1 complete. Error: 0.0 Total: 1.5765; Orientation: 0.0438; Line Search: 1.4783
Non-optimal measurement 0.0 < 0.0. Total: 2
LBFGS Accumulation History: 2 points
Non-optimal measurement 0.0 < 0.0. Total: 2
th(0)=0.0;dx=-7.22948938210888E27
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(2.154434690031884E-15)=0.0; dx=-7.229489382109016E27 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
Armijo: th(1.077217345015942E-15)=0.0; dx=-7.229489382108948E27 evalInputDelta=0.0
Non-optimal measurement 0.0 < 0.0. Total: 2
MIN ALPHA (3.5907244833864734E-16): th(0.0)=0.0
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.3779; Orientation: 0.1117; Line Search: 0.2525
Iteration 2 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 2
Final threshold in iteration 2: 0.0 (> 0.0) after 1.954s (< 30.000s)

Returns

    0.0

Training Converged

TrainingTester.java:432 executed in 0.01 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging
Plotting range=[0.0, 0.0], [2.0, 1.0]; valueStats=DoubleSummaryStatistics{count=0, sum=0.000000, min=Infinity, average=0.000000, max=-Infinity}
Only 0 points for GD
Only 0 points for CjGD
Only 0 points for LBFGS

Returns

Result

TrainingTester.java:435 executed in 0.00 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging
No Data

Results

TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):

    return grid(inputLearning, modelLearning, completeLearning);

Returns

Result

TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):

    return new ComponentResult(null == inputLearning ? null : inputLearning.value,
        null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);

Returns

    {"input":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "model":{ "LBFGS": { "type": "NonConverged", "value": NaN }, "CjGD": { "type": "NonConverged", "value": NaN }, "GD": { "type": "NonConverged", "value": NaN } }, "complete":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }}

LayerTests.java:425 executed in 0.00 seconds (0.000 gc):

    throwException(exceptions.addRef());

Results

detailsresult
{"input":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }, "model":{ "LBFGS": { "type": "NonConverged", "value": NaN }, "CjGD": { "type": "NonConverged", "value": NaN }, "GD": { "type": "NonConverged", "value": NaN } }, "complete":{ "LBFGS": { "type": "Converged", "value": 0.0 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 0.0 } }}OK
  {
    "result": "OK",
    "performance": {
      "execution_time": "24.419",
      "gc_time": "0.297"
    },
    "created_on": 1586739222893,
    "file_name": "trainingTest",
    "report": {
      "simpleName": "Basic",
      "canonicalName": "com.simiacryptus.mindseye.layers.java.LinearActivationLayerTest.Basic",
      "link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/test/java/com/simiacryptus/mindseye/layers/java/LinearActivationLayerTest.java",
      "javaDoc": ""
    },
    "training_analysis": {
      "input": {
        "LBFGS": {
          "type": "Converged",
          "value": 0.0
        },
        "CjGD": {
          "type": "Converged",
          "value": 0.0
        },
        "GD": {
          "type": "Converged",
          "value": 0.0
        }
      },
      "model": {
        "LBFGS": {
          "type": "NonConverged",
          "value": "NaN"
        },
        "CjGD": {
          "type": "NonConverged",
          "value": "NaN"
        },
        "GD": {
          "type": "NonConverged",
          "value": "NaN"
        }
      },
      "complete": {
        "LBFGS": {
          "type": "Converged",
          "value": 0.0
        },
        "CjGD": {
          "type": "Converged",
          "value": 0.0
        },
        "GD": {
          "type": "Converged",
          "value": 0.0
        }
      }
    },
    "archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/java/LinearActivationLayer/Basic/trainingTest/202004135342",
    "id": "42615725-d2f7-4641-a9d7-bef84374694e",
    "report_type": "Components",
    "display_name": "Comparative Training",
    "target": {
      "simpleName": "LinearActivationLayer",
      "canonicalName": "com.simiacryptus.mindseye.layers.java.LinearActivationLayer",
      "link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/main/java/com/simiacryptus/mindseye/layers/java/LinearActivationLayer.java",
      "javaDoc": ""
    }
  }