1. Test Modules
  2. Training Characteristics
    1. Input Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    2. Results
  3. Results

Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase

Test Modules

Using Seed 1926555871558952960

Training Characteristics

Input Learning

In this apply, we use a network to learn this target input, given it's pre-evaluated output:

TrainingTester.java:332 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
      try {
        return x.prettyPrint();
      } finally {
        x.freeRef();
      }
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [ 8.0, 49.6, -12.8, 70.0 ]
    [ 8.0, 49.6, 70.0, -12.8 ]
    [ 70.0, -12.8, 8.0, 49.6 ]
    [ 8.0, 70.0, -12.8, 49.6 ]
    [ 70.0, 8.0, 49.6, -12.8 ]

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:480 executed in 0.41 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging 
Reset training subject: 3967346030181
BACKPROP_AGG_SIZE = 3
THREADS = 64
SINGLE_THREADED = false
Initialized CoreSettings = {
"backpropAggregationSize" : 3,
"jvmThreads" : 64,
"singleThreaded" : false
}
Reset training subject: 3967380449623
Constructing line search parameters: GD
th(0)=10.53043196956189;dx=-3.6123014690479775E18
New Minimum: 10.53043196956189 > 1.8262491832550787
Armijo: th(2.154434690031884)=1.8262491832550787; dx=-8.521257045899994E-5 evalInputDelta=8.704182786306811
Armijo: th(1.077217345015942)=1.8263408329549413; dx=-8.475839977830507E-5 evalInputDelta=8.70409113660695
Armijo: th(0.3590724483386473)=1.8264006421672776; dx=-7.878088889413239E-5 evalInputDelta=8.704031327394613
Armijo: th(0.08976811208466183)=1.826416360400237; dx=2.333415908250027E-5 evalInputDelta=8.704015609161653
Armijo: th(0.017953622416932366)=1.8263833916172814; dx=0.0026375955931465245 evalInputDelta=8.70404857794461
New Minimum: 1.8262491832550787 > 1.8261403854227727
Armijo: th(0.002992270402822061)=1.8261403854227727; dx=0.0978130985870477 evalInputDelta=8.704291584139117
New Minimum: 1.8261403854227727 > 1.8243917730481918
Armijo: th(4.2746720040315154E-4)=1.8243917730481918; dx=4.749043020264634 evalInputDelta=8.7060401965137
New Minimum: 1.8243917730481918 > 1.8107327149772385
Armijo: th(5.343340005039394E-5)=1.8107327149772385; dx=280.9196238356807 evalInputDelta=8.719699254584652
New Minimum: 1.8107327149772385 > 1.7201421221589501
Armijo: th(5.9370444500437714E-6)=1.7201421221589501; dx=13161.71999806974 evalInputDelta=8.81028984740294
New Minimum: 1.7201421221589501 > 1.4877872689345701
Armijo: th(5.937044450043771E-7)=1.4877872689345701; dx=141945.00270125287 evalInputDelta=9.04264470062732
Armijo: th(5.397313136403428E-8)=1.9459420058127894; dx=-1.6007471744227428E16 evalInputDelta=8.584489963749101
Armijo: th(4.4977609470028565E-9)=10.536937112587974; dx=-3.2374119795468483E18 evalInputDelta=-0.00650514302608407
Armijo: th(3.4598161130791205E-10)=10.53093977782218; dx=-3.5811623931312251E18 evalInputDelta=-5.078082602896217E-4
Armijo: th(2.4712972236279432E-11)=10.530468283007947; dx=-3.6100635430687596E18 evalInputDelta=-3.631344605636855E-5
Armijo: th(1.6475314824186289E-12)=10.530434390656739; dx=-3.6121522079900764E18 evalInputDelta=-2.421094848159555E-6
Armijo: th(1.029707176511643E-13)=10.53043212088115; dx=-3.6122921399555456E18 evalInputDelta=-1.5131925934497303E-7
Armijo: th(6.057101038303783E-15)=10.530431978463026; dx=-3.6123009202768148E18 evalInputDelta=-8.901135828409679E-9
MIN ALPHA (3.3650561323909904E-16): th(5.937044450043771E-7)=1.4877872689345701
Fitness changed from 10.53043196956189 to 1.4877872689345701
Iteration 1 complete. Error: 1.4877872689345701 Total: 0.1653; Orientation: 0.0033; Line Search: 0.1148
th(0)=1.4877872689345701;dx=-9.019103804164644E-5
WOLFE (weak): th(2.154434690031884E-15)=1.4877872689345701; dx=-9.019103804164646E-5 evalInputDelta=0.0
WOLFE (weak): th(4.308869380063768E-15)=1.4877872689345701; dx=-9.019103804164644E-5 evalInputDelta=0.0
WOLFE (weak): th(1.2926608140191303E-14)=1.4877872689345701; dx=-9.019103804164646E-5 evalInputDelta=0.0
WOLFE (weak): th(5.1706432560765214E-14)=1.4877872689345701; dx=-9.019103804164646E-5 evalInputDelta=0.0
WOLFE (weak): th(2.5853216280382605E-13)=1.4877872689345701; dx=-9.019103804164643E-5 evalInputDelta=0.0
New Minimum: 1.4877872689345701 > 1.48778726893457
WOLFE (weak): th(1.5511929768229563E-12)=1.48778726893457; dx=-9.019103804164624E-5 evalInputDelta=2.220446049250313E-16
New Minimum: 1.48778726893457 > 1.487787268934569
WOLFE (weak): th(1.0858350837760695E-11)=1.487787268934569; dx=-9.01910380416453E-5 evalInputDelta=1.1102230246251565E-15
New Minimum: 1.487787268934569 > 1.4877872689345621
WOLFE (weak): th(8.686680670208556E-11)=1.4877872689345621; dx=-9.019103804163725E-5 evalInputDelta=7.993605777301127E-15
New Minimum: 1.4877872689345621 > 1.4877872689344995
WOLFE (weak): th(7.8180126031877E-10)=1.4877872689344995; dx=-9.019103804156387E-5 evalInputDelta=7.061018436615996E-14
New Minimum: 1.4877872689344995 > 1.487787268933865
WOLFE (weak): th(7.818012603187701E-9)=1.487787268933865; dx=-9.019103804082074E-5 evalInputDelta=7.052136652418994E-13
New Minimum: 1.487787268933865 > 1.487787268926814
WOLFE (weak): th(8.599813863506471E-8)=1.487787268926814; dx=-9.019103803256361E-5 evalInputDelta=7.756240094636269E-12
New Minimum: 1.487787268926814 > 1.4877872688414948
WOLFE (weak): th(1.0319776636207765E-6)=1.4877872688414948; dx=-9.019103793265238E-5 evalInputDelta=9.307532522484507E-11
New Minimum: 1.4877872688414948 > 1.4877872677245931
WOLFE (weak): th(1.3415709627070094E-5)=1.4877872677245931; dx=-9.01910366247235E-5 evalInputDelta=1.2099770074769367E-9
New Minimum: 1.4877872677245931 > 1.487787251994897
WOLFE (weak): th(1.878199347789813E-4)=1.487787251994897; dx=-9.019101820472897E-5 evalInputDelta=1.6939673219695806E-8
New Minimum: 1.487787251994897 > 1.487787014839866
WOLFE (weak): th(0.0028172990216847197)=1.487787014839866; dx=-9.019074048875518E-5 evalInputDelta=2.5409470416626334E-7
New Minimum: 1.487787014839866 > 1.4877832035198968
WOLFE (weak): th(0.045076784346955515)=1.4877832035198968; dx=-9.018627741930352E-5 evalInputDelta=4.065414673348755E-6
New Minimum: 1.4877832035198968 > 1.4877181860537507
WOLFE (weak): th(0.7663053338982437)=1.4877181860537507; dx=-9.011017237278707E-5 evalInputDelta=6.908288081941549E-5
New Minimum: 1.4877181860537507 > 1.4865531648870731
WOLFE (weak): th(13.793496010168386)=1.4865531648870731; dx=-8.875624343109561E-5 evalInputDelta=0.00123410404749702
New Minimum: 1.4865531648870731 > 1.4671969651158803
END: th(262.07642419319933)=1.4671969651158803; dx=-6.876792084097648E-5 evalInputDelta=0.02059030381868987
Fitness changed from 1.4877872689345701 to 1.4671969651158803
Iteration 2 complete. Error: 1.4671969651158803 Total: 0.0787; Orientation: 0.0008; Line Search: 0.0740
th(0)=1.4671969651158803;dx=-5.3135181304461636E-5
New Minimum: 1.4671969651158803 > 1.441560495860109
END: th(564.6265397213399)=1.441560495860109; dx=-3.9091797356454553E-5 evalInputDelta=0.025636469255771344
Fitness changed from 1.4671969651158803 to 1.441560495860109
Iteration 3 complete. Error: 1.441560495860109 Total: 0.0092; Orientation: 0.0008; Line Search: 0.0059
th(0)=1.441560495860109;dx=-2.9691957484091207E-5
New Minimum: 1.441560495860109 > 1.4103478252982034
END: th(1216.4510040883201)=1.4103478252982034; dx=-2.2393335817986673E-5 evalInputDelta=0.031212670561905576
Fitness changed from 1.441560495860109 to 1.4103478252982034
Iteration 4 complete. Error: 1.4103478252982034 Total: 0.0087; Orientation: 0.0007; Line Search: 0.0059
th(0)=1.4103478252982034;dx=-1.785806665863867E-5
New Minimum: 1.4103478252982034 > 1.3682850443897174
END: th(2620.7642419319936)=1.3682850443897174; dx=-1.4564621133295292E-5 evalInputDelta=0.04206278090848592
Fitness changed from 1.4103478252982034 to 1.3682850443897174
Iteration 5 complete. Error: 1.3682850443897174 Total: 0.0091; Orientation: 0.0007; Line Search: 0.0065
th(0)=1.3682850443897174;dx=-1.273515993072005E-5
New Minimum: 1.3682850443897174 > 1.2999234899173309
WOLFE (weak): th(5646.265397213399)=1.2999234899173309; dx=-1.1587336453602766E-5 evalInputDelta=0.06836155447238657
New Minimum: 1.2999234899173309 > 1.2365743575337476
END: th(11292.530794426799)=1.2365743575337476; dx=-1.0906369450669597E-5 evalInputDelta=0.13171068685596987
Fitness changed from 1.3682850443897174 to 1.2365743575337476
Iteration 6 complete. Error: 1.2365743575337476 Total: 0.0099; Orientation: 0.0007; Line Search: 0.0073
th(0)=1.2365743575337476;dx=-1.2199302937611788E-5
Armijo: th(24329.020081766404)=6.760359638613051; dx=-2.0326914771959538E9 evalInputDelta=-5.523785281079303
New Minimum: 1.2365743575337476 > 1.0633229696414723
WOLFE (weak): th(12164.510040883202)=1.0633229696414723; dx=-2.3760563137140976E-5 evalInputDelta=0.17325138789227523
New Minimum: 1.0633229696414723 > 1.0263295811961857
WOLFE (weak): th(18246.765061324804)=1.0263295811961857; dx=-9.632625997874891E-5 evalInputDelta=0.2102447763375619
New Minimum: 1.0263295811961857 > 1.0158813342860378
WOLFE (weak): th(21287.892571545606)=1.0158813342860378; dx=-6.034479925485284E-4 evalInputDelta=0.22069302324770979
New Minimum: 1.0158813342860378 > 1.013227806532993
WOLFE (weak): th(22808.456326656007)=1.013227806532993; dx=-0.011512106225308282 evalInputDelta=0.22334655100075462
Armijo: th(23568.738204211208)=6.760419761723206; dx=-2.4178796641699615E10 evalInputDelta=-5.5238454041894585
Armijo: th(23188.597265433607)=1.381876250699361; dx=1.2432667318745653E8 evalInputDelta=-0.14530189316561337
WOLFE (weak): th(22998.526796044807)=1.0158052181532324; dx=-0.03450957372445638 evalInputDelta=0.22076913938051512
mu ~= nu (22998.526796044807): th(22808.456326656007)=1.013227806532993
Fitness changed from 1.2365743575337476 to 1.013227806532993
Iteration 7 complete. Error: 1.013227806532993 Total: 0.0314; Orientation: 0.0008; Line Search: 0.0288
th(0)=1.013227806532993;dx=-27.757806550137634
Armijo: th(49753.57115542771)=1.4637434719060143; dx=251060.7051432977 evalInputDelta=-0.45051566537302135
Armijo: th(24876.785577713854)=1.2107731239107558; dx=-4.188097739146387E-6 evalInputDelta=-0.19754531737776282
Armijo: th(8292.261859237951)=1.2776591746066297; dx=-3.871673301967432E-6 evalInputDelta=-0.26443136807363676
Armijo: th(2073.065464809488)=1.302348983020476; dx=-4.579642667395483E-6 evalInputDelta=-0.28912117648748303
Armijo: th(414.61309296189756)=1.3160422715352467; dx=-2.7070156962077738E-5 evalInputDelta=-0.30281446500225373
Armijo: th(69.10218216031626)=1.3730557779770904; dx=-0.0011235332465177512 evalInputDelta=-0.3598279714440975
Armijo: th(9.871740308616609)=6.774713237294398; dx=-1.0130059601927631E10 evalInputDelta=-5.761485430761405
Armijo: th(1.233967538577076)=6.775535373981716; dx=-8.640264795010449E11 evalInputDelta=-5.762307567448723
New Minimum: 1.013227806532993 > 1.0055275797390424
WOLFE (weak): th(0.13710750428634177)=1.0055275797390424; dx=-412.6109265725756 evalInputDelta=0.007700226793950549
Armijo: th(0.685537521431709)=6.776574952733976; dx=-3.795603996382078E12 evalInputDelta=-5.763347146200983
Armijo: th(0.4113225128590254)=6.779327866813192; dx=-1.8575369182184406E13 evalInputDelta=-5.766100060280199
Armijo: th(0.27421500857268355)=6.792559502982774; dx=-1.1972568970479597E14 evalInputDelta=-5.779331696449781
Armijo: th(0.20566125642951266)=7.140492918322477; dx=-2.251815175161759E15 evalInputDelta=-6.127265111789484
New Minimum: 1.0055275797390424 > 0.9981266764274415
WOLFE (weak): th(0.17138438035792722)=0.9981266764274415; dx=-5043.971597475367 evalInputDelta=0.015101130105551475
Armijo: th(0.18852281839371993)=7.140492980643676; dx=-8.1796285196997472E16 evalInputDelta=-6.127265174110683
New Minimum: 0.9981266764274415 > 0.9981266452668457
WOLFE (weak): th(0.17995359937582356)=0.9981266452668457; dx=-35708.51619020358 evalInputDelta=0.015101161266147245
New Minimum: 0.9981266452668457 > 0.9981266296865474
WOLFE (weak): th(0.18423820888477174)=0.9981266296865474; dx=-1240857.0055312926 evalInputDelta=0.015101176846445519
Armijo: th(0.18638051363924585)=7.140492988433825; dx=-5.9227118781408077E17 evalInputDelta=-6.127265181900832
Armijo: th(0.1853093612620088)=7.140492992328899; dx=-2.479203328309673E19 evalInputDelta=-6.127265185795906
mu ~= nu (0.18423820888477174): th(0.18423820888477174)=0.9981266296865474
Fitness changed from 1.013227806532993 to 0.9981266296865474
Iteration 8 complete. Error: 0.9981266296865474 Total: 0.0455; Orientation: 0.0005; Line Search: 0.0437
th(0)=0.9981266296865474;dx=-5.547443103873112E10
Armijo: th(0.3980830523706075)=1.306521116564574; dx=-5.410171385509023E-4 evalInputDelta=-0.3083944868780266
Armijo: th(0.19904152618530374)=1.3067358757223613; dx=-0.0021554350053335698 evalInputDelta=-0.30860924603581386
Armijo: th(0.06634717539510125)=1.307594771746976; dx=-0.019452189596807144 evalInputDelta=-0.3094681420604286
Armijo: th(0.016586793848775313)=1.3115035628789187; dx=-0.31725609601612886 evalInputDelta=-0.31337693319237125
Armijo: th(0.0033173587697550625)=1.333734185073197; dx=-8.85696043213229 evalInputDelta=-0.33560755538664955
Armijo: th(5.528931282925105E-4)=1.6109934115741151; dx=-1437.1760534498612 evalInputDelta=-0.6128667818875677
Armijo: th(7.898473261321578E-5)=6.774970138595248; dx=-3.4096884078851405E15 evalInputDelta=-5.7768435089087005
Armijo: th(9.873091576651973E-6)=6.776926550111649; dx=-2.1897953915547712E17 evalInputDelta=-5.778799920425102
Armijo: th(1.0970101751835525E-6)=7.140492996219985; dx=-1.8319341658722791E19 evalInputDelta=-6.142366366533437
Armijo: th(1.0970101751835525E-7)=7.140492996223576; dx=-2.6183643109500744E21 evalInputDelta=-6.142366366537028
New Minimum: 0.9981266296865474 > 0.9981266296865112
Armijo: th(9.972819774395933E-9)=0.9981266296865112; dx=-2.3043956406289374E11 evalInputDelta=3.6193270602780103E-14
Armijo: th(8.310683145329943E-10)=0.9981266296865444; dx=-6.050154828227025E10 evalInputDelta=2.9976021664879227E-15
Armijo: th(6.392833188715341E-11)=0.9981266296865472; dx=-5.583847128263163E10 evalInputDelta=2.220446049250313E-16
Armijo: th(4.566309420510958E-12)=0.9981266296865474; dx=-5.550031568508705E10 evalInputDelta=0.0
Armijo: th(3.0442062803406384E-13)=0.9981266296865474; dx=-5.5476156118165375E10 evalInputDelta=0.0
Armijo: th(1.902628925212899E-14)=0.9981266296865474; dx=-5.54745388538811E10 evalInputDelta=0.0
Armijo: th(1.1191934854193523E-15)=0.9981266296865474; dx=-5.547443738062916E10 evalInputDelta=0.0
MIN ALPHA (6.217741585663068E-17): th(9.972819774395933E-9)=0.9981266296865112
Fitness changed from 0.9981266296865474 to 0.9981266296865112
Iteration 9 complete. Error: 0.9981266296865112 Total: 0.0382; Orientation: 0.0004; Line Search: 0.0362
th(0)=0.9981266296865112;dx=-9.572408715312329E11
Armijo: th(2.154434690031884E-15)=0.9981266296865112; dx=-9.572426550715731E11 evalInputDelta=0.0
Armijo: th(1.077217345015942E-15)=0.9981266296865112; dx=-9.572417633007799E11 evalInputDelta=0.0
MIN ALPHA (3.5907244833864734E-16): th(0.0)=0.9981266296865112
Fitness changed from 0.9981266296865112 to 0.9981266296865112
Static Iteration Total: 0.0079; Orientation: 0.0006; Line Search: 0.0059
Iteration 10 failed. Error: 0.9981266296865112
Previous Error: 0.0 -> 0.9981266296865112
Optimization terminated 10
Final threshold in iteration 10: 0.9981266296865112 (> 0.0) after 0.405s (< 30.000s)

Returns

    0.9981266296865112

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

And regressed input:

TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [ 135.85566664208895, 108.35537350856679, 111.5700149881933, 686.7314575005917 ]

To produce the following output:

TrainingTester.java:633 executed in 0.00 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [ 0.13031562211002012, 0.10393676065741586, 0.10702031259626757, 0.6587273046362964 ]

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:452 executed in 0.92 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging 
Reset training subject: 3967767253044
Reset training subject: 3967768931858
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=10.53043196956189}, derivative=-3.6123014690479775E18}
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=10.53057887108364}, derivative=-3.6032588481395072E18}, evalInputDelta = 1.469015217487879E-4
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=10.531458090900951}, derivative=-3.5497228683216655E18}, evalInputDelta = 0.0010261213390609925
F(5.384615384615385E-11) = LineSearchPoint{point=PointSample{avg=10.53051108335536}, derivative=-3.6074280591483418E18}, evalInputDelta = 7.911379347014247E-5
F(3.7692307692307697E-10) = LineSearchPoint{point=PointSample{avg=10.53098513079542}, derivative=-3.5783976077066424E18}, evalInputDelta = 5.531612335296643E-4
F(2.899408284023669E-11) = LineSearchPoint{point=PointSample{avg=10.530474573059166}, derivative=-3.6096760751444726E18}, evalInputDelta = 4.2603497275806035E-5
F(2.0295857988165684E-10) = LineSearchPoint{point=PointSample{avg=10.530730009752258}, derivative=-3.5939848100619581E18}, evalInputDelta = 2.9804019036738794E-4
F(1.561219845243514E-11) = LineSearchPoint{point=PointSample{avg=10.530454910997525}, derivative=-3.6108874327101819E18}, evalInputDelta = 2.294143563474904E-5
F(1.0928538916704598E-10) = LineSearchPoint{point=PointSample{avg=10.530592506164746}, derivative=-3.6024209622756849E18}, evalInputDelta = 1.605366028556432E-4
F(8.406568397465076E-12) = LineSearchPoint{point=PointSample{avg=10.530444322958942}, derivative=-3.6115399596679296E18}, evalInputDelta = 1.2353397051967363E-5
F(5.884597878225553E-11) = LineSearchPoint{point=PointSample{avg=10.530518427842846}, derivative=-3.6069760542550016E18}, evalInputDelta = 8.645828095588115E-5
F(4.119218514757887E-10) = LineSearchPoint{point=PointSample{avg=10.531036418816939}, derivative=-3.5752742120648392E18}, evalInputDelta = 6.044492550483227E-4
F(3.168629626736836E-11) = LineSearchPoint{point=PointSample{avg=10.530478528514248}, derivative=-3.6094324451585751E18}, evalInputDelta = 4.6558952357500516E-5
F(2.2180407387157852E-10) = LineSearchPoint{point=PointSample{avg=10.530757662135702}, derivative=-3.592291246338281E18}, evalInputDelta = 3.256925738117644E-4
F(1.706185183627527E-11) = LineSearchPoint{point=PointSample{avg=10.530457041069946}, derivative=-3.6107561768812498E18}, evalInputDelta = 2.507150805541869E-5
F(1.194329628539269E-10) = LineSearchPoint{point=PointSample{avg=10.53060740628714}, derivative=-3.6015056161209221E18}, evalInputDelta = 1.7543672525022203E-4
F(9.187150988763607E-12) = LineSearchPoint{point=PointSample{avg=10.530445469982485}, derivative=-3.6114692630140677E18}, evalInputDelta = 1.3500420594780849E-5
F(6.431005692134525E-11) = LineSearchPoint{point=PointSample{avg=10.530526453996107}, derivative=-3.6064821779817257E18}, evalInputDelta = 9.44844342161133E-5
F(4.501703984494168E-10) = LineSearchPoint{point=PointSample{avg=10.531092454542087}, derivative=-3.5718655617745418E18}, evalInputDelta = 6.604849801963297E-4
F(3.4628492188416676E-11) = LineSearchPoint{point=PointSample{avg=10.530482851162173}, derivative=-3.6091662219346872E18}, evalInputDelta = 5.088160028243749E-5
F(2.423994453189167E-10) = LineSearchPoint{point=PointSample{avg=10.530787877914996}, derivative=-3.5904418237817196E18}, evalInputDelta = 3.5590835310550517E-4
F(1.864611117837821E-11) = LineSearchPoint{point=PointSample{avg=10.53045936890278}, derivative=-3.6106127417811809E18}, evalInputDelta = 2.7399340890355006E-5
F(1.3052277824864746E-10) = LineSearchPoint{point=PointSample{avg=10.530623688714941}, derivative=-3.6005056831745459E18}, evalInputDelta = 1.917191530509399E-4
F(1.004021371143442E-11) = LineSearchPoint{point=PointSample{avg=10.53044672350435}, derivative=-3.6113920043208417E18}, evalInputDelta = 1.4753942458867186E-5
F(7.028149598004095E-11) = LineSearchPoint{point=PointSample{avg=10.530535225052471}, derivative=-3.6059425616195553E18}, evalInputDelta = 1.0325549058087802E-4
F(4.919704718602866E-10) = LineSearchPoint{point=PointSample{avg=10.531153675983347}, derivative=-3.5681460845504778E18}, evalInputDelta = 7.21706421456858E-4
F(3.784388245079128E-11) = LineSearchPoint{point=PointSample{avg=10.530487575081406}, derivative=-3.6088753131853138E18}, evalInputDelta = 5.560551951511172E-5
F(2.64907177155539E-10) = LineSearchPoint{point=PointSample{avg=10.530820894289052}, derivative=-3.5884223396372132E18}, evalInputDelta = 3.889247271615659E-4
F(2.037747516581069E-11) = LineSearchPoint{point=PointSample{avg=10.53046191285398}, derivative=-3.6104559981377976E18}, evalInputDelta = 2.9943292089384954E-5
F(1.4264232616067484E-10) = LineSearchPoint{point=PointSample{avg=10.530641481561874}, derivative=-3.5994133879710372E18}, evalInputDelta = 2.09511999983647E-4
F(1.0972486627744219E-11) = LineSearchPoint{point=PointSample{avg=10.530448093411906}, derivative=-3.6113075747475256E18}, evalInputDelta = 1.6123850015148378E-5
F(7.680740639420954E-11) = LineSearchPoint{point=PointSample{avg=10.530544810109843}, derivative=-3.6053529809580288E18}, evalInputDelta = 1.1284054795268617E-4
F(5.376518447594668E-10) = LineSearchPoint{point=PointSample{avg=10.53122056127582}, derivative=-3.5640880079912489E18}, evalInputDelta = 7.885917139294207E-4
F(4.135783421226668E-11) = LineSearchPoint{point=PointSample{avg=10.530492737511466}, derivative=-3.6085574334291825E18}, evalInputDelta = 6.076794957543541E-5
F(2.8950483948586675E-10) = LineSearchPoint{point=PointSample{avg=10.530856970322043}, derivative=-3.586217324171714E18}, evalInputDelta = 4.2500076015272725E-4
F(2.2269603037374367E-11) = LineSearchPoint{point=PointSample{avg=10.530464692985145}, derivative=-3.6102847121362801E18}, evalInputDelta = 3.272342325466582E-5
F(1.5588722126162056E-10) = LineSearchPoint{point=PointSample{avg=10.530660924791178}, derivative=-3.5982202482806794E18}, evalInputDelta = 2.2895522928756407E-4
F(1.1991324712432351E-11) = LineSearchPoint{point=PointSample{avg=10.530449590510328}, derivative=-3.6112153090114877E18}, evalInputDelta = 1.7620948437269135E-5
F(8.393927298702646E-11) = LineSearchPoint{point=PointSample{avg=10.53055528466868}, derivative=-3.6047088239958876E18}, evalInputDelta = 1.2331510678897928E-4
F(5.875749109091853E-10) = LineSearchPoint{point=PointSample{avg=10.53129363229662}, derivative=-3.5596611881575644E18}, evalInputDelta = 8.616627347297623E-4
F(4.5198070069937326E-11) = LineSearchPoint{point=PointSample{avg=10.5304983791458}, derivative=-3.6082100862624727E18}, evalInputDelta = 6.640958391024299E-5
F(3.1638649048956127E-10) = LineSearchPoint{point=PointSample{avg=10.530896388942736}, derivative=-3.5838099329342464E18}, evalInputDelta = 4.644193808456265E-4
F(2.4337422345350867E-11) = LineSearchPoint{point=PointSample{avg=10.530467731219554}, derivative=-3.6100975357730437E18}, evalInputDelta = 3.576165766361328E-5
F(1.7036195641745607E-10) = LineSearchPoint{point=PointSample{avg=10.530682171306868}, derivative=-3.5969170124152694E18}, evalInputDelta = 2.5020174497747405E-4
F(1.3104765878265852E-11) = LineSearchPoint{point=PointSample{avg=10.530451226607768}, derivative=-3.6111144801650811E18}, evalInputDelta = 1.925704587790733E-5
F(9.173336114786096E-11) = LineSearchPoint{point=PointSample{avg=10.53056673122388}, derivative=-3.6040050557926605E18}, evalInputDelta = 1.3476166198955752E-4
F(6.421335280350267E-10) = LineSearchPoint{point=PointSample{avg=10.53137345859987}, derivative=-3.5548329283165819E18}, evalInputDelta = 9.414890379790819E-4
F(4.939488677192513E-11) = LineSearchPoint{point=PointSample{avg=10.530504544451741}, derivative=-3.6078305450245294E18}, evalInputDelta = 7.257488985068505E-5
F(3.457642074034759E-10) = LineSearchPoint{point=PointSample{avg=10.530939459123466}, derivative=-3.5811818308873779E18}, evalInputDelta = 5.0748956157598E-4
F(2.65972467233443E-11) = LineSearchPoint{point=PointSample{avg=10.530471051514786}, derivative=-3.6098929963257846E18}, evalInputDelta = 3.9081952895259064E-5
F(1.861807270634101E-10) = LineSearchPoint{point=PointSample{avg=10.530705388144597}, derivative=-3.5954935912742922E18}, evalInputDe

...skipping 68796 bytes...

SearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-4.6470871589837236E24}, evalInputDelta = 6.142366366537426
F(6.869138356585477E-10) = LineSearchPoint{point=PointSample{avg=6.909145296383078}, derivative=-9.241514204952603E22}, evalInputDelta = 6.1423663665374235
F(5.2839525819888284E-11) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-1.645358154885973E25}, evalInputDelta = 6.142366366537426
F(3.69876680739218E-10) = LineSearchPoint{point=PointSample{avg=6.909145296383079}, derivative=-3.1991221507649135E23}, evalInputDelta = 6.142366366537424
F(2.8452052364555228E-11) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-5.963574212632185E25}, evalInputDelta = 6.142366366537426
F(1.991643665518866E-10) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-1.1109518454924845E24}, evalInputDelta = 6.142366366537426
F(1.532033588860666E-11) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-2.2631121607580137E26}, evalInputDelta = 6.142366366537426
F(1.0724235122024664E-10) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-3.880929602968543E24}, evalInputDelta = 6.142366366537426
F(8.249411632326664E-12) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-9.444492084230088E26}, evalInputDelta = 6.142366366537426
F(5.774588142628665E-11) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-1.3709713973760348E25}, evalInputDelta = 6.142366366537426
F(4.042211699840066E-10) = LineSearchPoint{point=PointSample{avg=6.909145296383079}, derivative=-2.6767782373504945E23}, evalInputDelta = 6.142366366537424
F(3.109393615261589E-11) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-4.947364131999389E25}, evalInputDelta = 6.142366366537426
F(2.1765755306831124E-10) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-9.29015198247517E23}, evalInputDelta = 6.142366366537426
F(1.6742888697562404E-11) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-1.8611871197891753E26}, evalInputDelta = 6.142366366537426
F(1.1720022088293684E-10) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-3.2418074059978656E24}, evalInputDelta = 6.142366366537426
F(9.015401606379756E-12) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-7.624487131642626E26}, evalInputDelta = 6.142366366537426
F(6.31078112446583E-11) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-1.142825452629623E25}, evalInputDelta = 6.142366366537426
F(4.4175467871260806E-10) = LineSearchPoint{point=PointSample{avg=6.909145296383079}, derivative=-2.2398505847098963E23}, evalInputDelta = 6.142366366537424
F(3.398112913173908E-11) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-4.107691761218227E25}, evalInputDelta = 6.142366366537426
F(2.3786790392217353E-10) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-7.769575378245038E23}, evalInputDelta = 6.142366366537426
F(1.8297531070936425E-11) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-1.5331927499475958E26}, evalInputDelta = 6.142366366537426
F(1.2808271749655497E-10) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-2.7084869396271194E24}, evalInputDelta = 6.142366366537426
F(9.852516730504228E-12) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-6.177855692711993E26}, evalInputDelta = 6.142366366537426
F(6.896761711352959E-11) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-9.530125893454015E24}, evalInputDelta = 6.142366366537426
F(4.827733197947071E-10) = LineSearchPoint{point=PointSample{avg=6.909145296383079}, derivative=-1.874341169711282E23}, evalInputDelta = 6.142366366537424
F(3.713640921497747E-11) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-3.413072333254169E25}, evalInputDelta = 6.142366366537426
F(2.599548645048423E-10) = LineSearchPoint{point=PointSample{avg=6.909145296383079}, derivative=-6.498522074713822E23}, evalInputDelta = 6.142366366537424
F(1.9996528038834024E-11) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-1.2648939645242958E26}, evalInputDelta = 6.142366366537426
F(1.3997569627183816E-10) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-2.263323921231603E24}, evalInputDelta = 6.142366366537426
F(1.0767361251679859E-11) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-5.022005607430363E26}, evalInputDelta = 6.142366366537426
F(7.537152876175901E-11) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-7.950048302813789E24}, evalInputDelta = 6.142366366537426
F(5.276007013323131E-10) = LineSearchPoint{point=PointSample{avg=6.909145296383079}, derivative=-1.5685530796210766E23}, evalInputDelta = 6.142366366537424
F(4.058466933325485E-11) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-2.8378343780092927E25}, evalInputDelta = 6.142366366537426
F(2.84092685332784E-10) = LineSearchPoint{point=PointSample{avg=6.909145296383079}, derivative=-5.435895247578793E23}, evalInputDelta = 6.142366366537424
F(2.1853283487137228E-11) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-1.0449562094495501E26}, evalInputDelta = 6.142366366537426
F(1.529729844099606E-10) = LineSearchPoint{point=PointSample{avg=6.90914529638308}, derivative=-1.89164716716341E24}, evalInputDelta = 6.142366366537426
Loops = 52
F(3.059459688199212E-10) = LineSearchPoint{point=PointSample{avg=6.909145296383079}, derivative=-4.6835945301364476E23}, evalInputDelta = 6.142366366537424
Right bracket at 3.059459688199212E-10
F(6.118919376398424E-10) = LineSearchPoint{point=PointSample{avg=6.909145296383078}, derivative=-1.1652697346684596E23}, evalInputDelta = 6.1423663665374235
Right bracket at 6.118919376398424E-10
F(1.2237838752796847E-9) = LineSearchPoint{point=PointSample{avg=6.909145296383075}, derivative=-2.9061762369802727E22}, evalInputDelta = 6.142366366537421
Right bracket at 1.2237838752796847E-9
F(2.4475677505593695E-9) = LineSearchPoint{point=PointSample{avg=6.909145296383069}, derivative=-7.256716598858432E21}, evalInputDelta = 6.142366366537415
Right bracket at 2.4475677505593695E-9
F(4.895135501118739E-9) = LineSearchPoint{point=PointSample{avg=6.909145296383056}, derivative=-1.8130901233176126E21}, evalInputDelta = 6.142366366537402
Right bracket at 4.895135501118739E-9
F(9.790271002237478E-9) = LineSearchPoint{point=PointSample{avg=6.909145296383032}, derivative=-4.5313649445099327E20}, evalInputDelta = 6.142366366537378
Right bracket at 9.790271002237478E-9
F(1.9580542004474956E-8) = LineSearchPoint{point=PointSample{avg=6.909145296382983}, derivative=-1.132671248064992E20}, evalInputDelta = 6.1423663665373285
Right bracket at 1.9580542004474956E-8
F(3.916108400894991E-8) = LineSearchPoint{point=PointSample{avg=6.909145296382883}, derivative=-2.831465670953365E19}, evalInputDelta = 6.142366366537229
Right bracket at 3.916108400894991E-8
F(7.832216801789982E-8) = LineSearchPoint{point=PointSample{avg=6.909145296382685}, derivative=-7.078398638286252E18}, evalInputDelta = 6.142366366537031
Right bracket at 7.832216801789982E-8
F(1.5664433603579965E-7) = LineSearchPoint{point=PointSample{avg=6.90914529638229}, derivative=-1.76956646858518528E18}, evalInputDelta = 6.142366366536636
Right bracket at 1.5664433603579965E-7
F(3.132886720715993E-7) = LineSearchPoint{point=PointSample{avg=6.558531393387744}, derivative=-4.3501179378673242E17}, evalInputDelta = 5.79175246354209
Right bracket at 3.132886720715993E-7
F(6.265773441431986E-7) = LineSearchPoint{point=PointSample{avg=4.555211386190466}, derivative=-7.5720692217654128E16}, evalInputDelta = 3.7884324563448115
Loops = 12
Fitness changed from 0.7667789298456545 to 0.7667789298456545
Static Iteration Total: 0.1161; Orientation: 0.0003; Line Search: 0.1149
Iteration 10 failed. Error: 0.7667789298456545
Previous Error: 0.0 -> 0.7667789298456545
Optimization terminated 10
Final threshold in iteration 10: 0.7667789298456545 (> 0.0) after 0.918s (< 30.000s)

Returns

    0.7667789298456545

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

And regressed input:

TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [ 25.85748358071801, 232.61829448486316, -24.65986535366525, 148.30916664149814 ]

To produce the following output:

TrainingTester.java:633 executed in 0.00 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [ 0.06766759100049367, 0.6087490904247158, -0.0645334909590119, 0.3881168095338024 ]

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:509 executed in 10.02 seconds (0.640 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging 
Reset training subject: 3968695829234
Reset training subject: 3968696734162
Adding measurement 6c7e57be to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 10.53043196956189 < 10.53043196956189. Total: 1
th(0)=10.53043196956189;dx=-3.6123014690479775E18
Adding measurement 7889aeb3 to history. Total: 1
New Minimum: 10.53043196956189 > 1.8262491832550787
Armijo: th(2.154434690031884)=1.8262491832550787; dx=-8.521257045899994E-5 evalInputDelta=8.704182786306811
Non-optimal measurement 1.8263408329549413 < 1.8262491832550787. Total: 2
Armijo: th(1.077217345015942)=1.8263408329549413; dx=-8.475839977830507E-5 evalInputDelta=8.70409113660695
Non-optimal measurement 1.8264006421672776 < 1.8262491832550787. Total: 2
Armijo: th(0.3590724483386473)=1.8264006421672776; dx=-7.878088889413239E-5 evalInputDelta=8.704031327394613
Non-optimal measurement 1.826416360400237 < 1.8262491832550787. Total: 2
Armijo: th(0.08976811208466183)=1.826416360400237; dx=2.333415908250027E-5 evalInputDelta=8.704015609161653
Non-optimal measurement 1.8263833916172814 < 1.8262491832550787. Total: 2
Armijo: th(0.017953622416932366)=1.8263833916172814; dx=0.0026375955931465245 evalInputDelta=8.70404857794461
Adding measurement 76e6f39c to history. Total: 2
New Minimum: 1.8262491832550787 > 1.8261403854227727
Armijo: th(0.002992270402822061)=1.8261403854227727; dx=0.0978130985870477 evalInputDelta=8.704291584139117
Adding measurement 78428fd6 to history. Total: 3
New Minimum: 1.8261403854227727 > 1.8243917730481918
Armijo: th(4.2746720040315154E-4)=1.8243917730481918; dx=4.749043020264634 evalInputDelta=8.7060401965137
Adding measurement ecfd08b to history. Total: 4
New Minimum: 1.8243917730481918 > 1.8107327149772385
Armijo: th(5.343340005039394E-5)=1.8107327149772385; dx=280.9196238356807 evalInputDelta=8.719699254584652
Adding measurement 3226dc25 to history. Total: 5
New Minimum: 1.8107327149772385 > 1.7201421221589501
Armijo: th(5.9370444500437714E-6)=1.7201421221589501; dx=13161.71999806974 evalInputDelta=8.81028984740294
Adding measurement 5ce4cc05 to history. Total: 6
New Minimum: 1.7201421221589501 > 1.4877872689345701
Armijo: th(5.937044450043771E-7)=1.4877872689345701; dx=141945.00270125287 evalInputDelta=9.04264470062732
Non-optimal measurement 1.9459420058127894 < 1.4877872689345701. Total: 7
Armijo: th(5.397313136403428E-8)=1.9459420058127894; dx=-1.6007471744227428E16 evalInputDelta=8.584489963749101
Non-optimal measurement 10.536937112587974 < 1.4877872689345701. Total: 7
Armijo: th(4.4977609470028565E-9)=10.536937112587974; dx=-3.2374119795468483E18 evalInputDelta=-0.00650514302608407
Non-optimal measurement 10.53093977782218 < 1.4877872689345701. Total: 7
Armijo: th(3.4598161130791205E-10)=10.53093977782218; dx=-3.5811623931312251E18 evalInputDelta=-5.078082602896217E-4
Non-optimal measurement 10.530468283007947 < 1.4877872689345701. Total: 7
Armijo: th(2.4712972236279432E-11)=10.530468283007947; dx=-3.6100635430687596E18 evalInputDelta=-3.631344605636855E-5
Non-optimal measurement 10.530434390656739 < 1.4877872689345701. Total: 7
Armijo: th(1.6475314824186289E-12)=10.530434390656739; dx=-3.6121522079900764E18 evalInputDelta=-2.421094848159555E-6
Non-optimal measurement 10.53043212088115 < 1.4877872689345701. Total: 7
Armijo: th(1.029707176511643E-13)=10.53043212088115; dx=-3.6122921399555451E18 evalInputDelta=-1.5131925934497303E-7
Non-optimal measurement 10.530431978463026 < 1.4877872689345701. Total: 7
Armijo: th(6.057101038303783E-15)=10.530431978463026; dx=-3.6123009202768148E18 evalInputDelta=-8.901135828409679E-9
Non-optimal measurement 1.4877872689345701 < 1.4877872689345701. Total: 7
MIN ALPHA (3.3650561323909904E-16): th(5.937044450043771E-7)=1.4877872689345701
Fitness changed from 10.53043196956189 to 1.4877872689345701
Iteration 1 complete. Error: 1.4877872689345701 Total: 0.0411; Orientation: 0.0026; Line Search: 0.0362
Non-optimal measurement 1.4877872689345701 < 1.4877872689345701. Total: 7
Rejected: LBFGS Orientation magnitude: 2.046e-02, gradient 9.497e-03, dot -1.000; [58f72121-9cb2-46b9-b925-6e0c4e592981 = 1.000/1.000e+00, 32293749-2561-45aa-b788-df6153a6b965 = 1.000/1.000e+00, f4566f37-3630-46d3-b16f-e2e3664dfea4 = 1.000/1.000e+00, 0cc7b127-97a6-4fa3-8b64-b517fdc17974 = 1.000/1.000e+00, 472dd6db-3165-464a-92be-e733a6431524 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.4877872689345701, 1.7201421221589501, 1.8107327149772385, 1.8243917730481918, 1.8261403854227727, 1.8262491832550787, 10.53043196956189
Accepted: LBFGS Orientation magnitude: 1.186e+16, gradient 9.497e-03, dot 0.008; [32293749-2561-45aa-b788-df6153a6b965 = 1.000/1.000e+00, 472dd6db-3165-464a-92be-e733a6431524 = 1.000/1.000e+00, 0cc7b127-97a6-4fa3-8b64-b517fdc17974 = 1.000/1.000e+00, 58f72121-9cb2-46b9-b925-6e0c4e592981 = 1.000/1.000e+00, f4566f37-3630-46d3-b16f-e2e3664dfea4 = 1.000/1.000e+00]
Overwriting history with 6 points
Constructing line search parameters: LBFGS
Non-optimal measurement 1.4877872689345701 < 1.4877872689345701. Total: 6
th(0)=1.4877872689345701;dx=-8.853605559793269E11
Non-optimal measurement 5.241755669078137 < 1.4877872689345701. Total: 6
Armijo: th(2.154434690031884)=5.241755669078137; dx=-133.12404304742816 evalInputDelta=-3.753968400143567
Non-optimal measurement 5.241755669078202 < 1.4877872689345701. Total: 6
Armijo: th(1.077217345015942)=5.241755669078202; dx=-532.4961770772935 evalInputDelta=-3.753968400143632
Non-optimal measurement 5.241755669078464 < 1.4877872689345701. Total: 6
Armijo: th(0.3590724483386473)=5.241755669078464; dx=-4792.465774774552 evalInputDelta=-3.753968400143894
Non-optimal measurement 5.241755669079639 < 1.4877872689345701. Total: 6
Armijo: th(0.08976811208466183)=5.241755669079639; dx=-76679.46515178682 evalInputDelta=-3.753968400145069
Non-optimal measurement 5.2417556690859115 < 1.4877872689345701. Total: 6
Armijo: th(0.017953622416932366)=5.2417556690859115; dx=-1916988.3299865727 evalInputDelta=-3.7539684001513414
Non-optimal measurement 5.24175566912511 < 1.4877872689345701. Total: 6
Armijo: th(0.002992270402822061)=5.24175566912511; dx=-6.901196264086916E7 evalInputDelta=-3.7539684001905402
Non-optimal measurement 5.241755669407338 < 1.4877872689345701. Total: 6
Armijo: th(4.2746720040315154E-4)=5.241755669407338; dx=-3.381721212077637E9 evalInputDelta=-3.753968400472768
Non-optimal measurement 5.241755671712204 < 1.4877872689345701. Total: 6
Armijo: th(5.343340005039394E-5)=5.241755671712204; dx=-2.165007592593848E11 evalInputDelta=-3.7539684027776334
Non-optimal measurement 5.241755692785259 < 1.4877872689345701. Total: 6
Armijo: th(5.9370444500437714E-6)=5.241755692785259; dx=-1.7588977091227957E13 evalInputDelta=-3.7539684238506887
Non-optimal measurement 5.241755906150121 < 1.4877872689345701. Total: 6
Armijo: th(5.937044450043771E-7)=5.241755906150121; dx=-1.8133185499119652E15 evalInputDelta=-3.7539686372155505
Non-optimal measurement 5.241758276892517 < 1.4877872689345701. Total: 6
Armijo: th(5.397313136403428E-8)=5.241758276892517; dx=-3.1894201194694554E17 evalInputDelta=-3.7539710079579467
Non-optimal measurement 3.9029608377479716 < 1.4877872689345701. Total: 6
Armijo: th(4.4977609470028565E-9)=3.9029608377479716; dx=2.7519144263961215E19 evalInputDelta=-2.4151735688134015
Non-optimal measurement 1.8805708196141684 < 1.4877872689345701. Total: 6
Armijo: th(3.4598161130791205E-10)=1.8805708196141684; dx=2.3555937973501816E8 evalInputDelta=-0.39278355067959825
Non-optimal measurement 1.8348948492191526 < 1.4877872689345701. Total: 6
Armijo: th(2.4712972236279432E-11)=1.8348948492191526; dx=-1.3046226952081078E8 evalInputDelta=-0.34710758028458244
Non-optimal measurement 1.9536649772293235 < 1.4877872689345701. Total: 6
Armijo: th(1.6475314824186289E-12)=1.9536649772293235; dx=-1.1921162960276117E11 evalInputDelta=-0.4658777082947534
Non-optimal measurement 10.473344837097839 < 1.4877872689345701. Total: 6
Armijo: th(1.029707176511643E-13)=10.473344837097839; dx=1.6255980588955116E26 evalInputDelta=-8.98555756816327
Adding measurement 3c5216d4 to history. Total: 6
New Minimum: 1.4877872689345701 > 1.48238078

...skipping 149112 bytes...

LBFGS Orientation magnitude: 8.229e+00, gradient 4.830e-03, dot -0.946; [f4566f37-3630-46d3-b16f-e2e3664dfea4 = 1.000/1.000e+00, 472dd6db-3165-464a-92be-e733a6431524 = 1.000/1.000e+00, 0cc7b127-97a6-4fa3-8b64-b517fdc17974 = 1.000/1.000e+00, 32293749-2561-45aa-b788-df6153a6b965 = 1.000/1.000e+00, 58f72121-9cb2-46b9-b925-6e0c4e592981 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.355484154513933, 1.3940205032093478, 1.4457195202954205, 1.4457195202954216, 1.4457195202954218
Rejected: LBFGS Orientation magnitude: 9.226e+00, gradient 4.830e-03, dot -0.975; [0cc7b127-97a6-4fa3-8b64-b517fdc17974 = 1.000/1.000e+00, 58f72121-9cb2-46b9-b925-6e0c4e592981 = 1.000/1.000e+00, 32293749-2561-45aa-b788-df6153a6b965 = 1.000/1.000e+00, 472dd6db-3165-464a-92be-e733a6431524 = 1.000/1.000e+00, f4566f37-3630-46d3-b16f-e2e3664dfea4 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.355484154513933, 1.3940205032093478, 1.4457195202954205, 1.4457195202954216
LBFGS Accumulation History: 3 points
Removed measurement de7086c to history. Total: 4
Removed measurement 55eb92bf to history. Total: 3
Adding measurement 47ffa568 to history. Total: 3
th(0)=1.355484154513933;dx=-2.332608145933055E-5
Adding measurement a7725e8 to history. Total: 4
New Minimum: 1.355484154513933 > 1.2999158603115473
END: th(2620.7642419319936)=1.2999158603115473; dx=-1.9425035426696507E-5 evalInputDelta=0.055568294202385626
Fitness changed from 1.355484154513933 to 1.2999158603115473
Iteration 104 complete. Error: 1.2999158603115473 Total: 0.0232; Orientation: 0.0206; Line Search: 0.0020
Non-optimal measurement 1.2999158603115473 < 1.2999158603115473. Total: 5
Rejected: LBFGS Orientation magnitude: 1.940e+01, gradient 4.132e-03, dot -0.956; [0cc7b127-97a6-4fa3-8b64-b517fdc17974 = 1.000/1.000e+00, f4566f37-3630-46d3-b16f-e2e3664dfea4 = 1.000/1.000e+00, 32293749-2561-45aa-b788-df6153a6b965 = 1.000/1.000e+00, 472dd6db-3165-464a-92be-e733a6431524 = 1.000/1.000e+00, 58f72121-9cb2-46b9-b925-6e0c4e592981 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.2999158603115473, 1.355484154513933, 1.4457195202954205, 1.4457195202954216, 1.4457195202954218
Rejected: LBFGS Orientation magnitude: 1.960e+01, gradient 4.132e-03, dot -0.969; [32293749-2561-45aa-b788-df6153a6b965 = 1.000/1.000e+00, 0cc7b127-97a6-4fa3-8b64-b517fdc17974 = 1.000/1.000e+00, 472dd6db-3165-464a-92be-e733a6431524 = 1.000/1.000e+00, f4566f37-3630-46d3-b16f-e2e3664dfea4 = 1.000/1.000e+00, 58f72121-9cb2-46b9-b925-6e0c4e592981 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.2999158603115473, 1.355484154513933, 1.4457195202954205, 1.4457195202954216
LBFGS Accumulation History: 3 points
Removed measurement a7725e8 to history. Total: 4
Removed measurement 47ffa568 to history. Total: 3
Adding measurement 56be778 to history. Total: 3
th(0)=1.2999158603115473;dx=-1.7077153378964514E-5
Adding measurement 6dab09c0 to history. Total: 4
New Minimum: 1.2999158603115473 > 1.2091506958428089
END: th(5646.265397213399)=1.2091506958428089; dx=-1.5220681606799946E-5 evalInputDelta=0.09076516446873839
Fitness changed from 1.2999158603115473 to 1.2091506958428089
Iteration 105 complete. Error: 1.2091506958428089 Total: 0.0236; Orientation: 0.0210; Line Search: 0.0020
Non-optimal measurement 1.2091506958428089 < 1.2091506958428089. Total: 5
Rejected: LBFGS Orientation magnitude: 8.025e+01, gradient 3.820e-03, dot -0.988; [32293749-2561-45aa-b788-df6153a6b965 = 1.000/1.000e+00, f4566f37-3630-46d3-b16f-e2e3664dfea4 = 1.000/1.000e+00, 472dd6db-3165-464a-92be-e733a6431524 = 1.000/1.000e+00, 0cc7b127-97a6-4fa3-8b64-b517fdc17974 = 1.000/1.000e+00, 58f72121-9cb2-46b9-b925-6e0c4e592981 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.2091506958428089, 1.2999158603115473, 1.4457195202954205, 1.4457195202954216, 1.4457195202954218
Rejected: LBFGS Orientation magnitude: 7.966e+01, gradient 3.820e-03, dot -0.988; [f4566f37-3630-46d3-b16f-e2e3664dfea4 = 1.000/1.000e+00, 0cc7b127-97a6-4fa3-8b64-b517fdc17974 = 1.000/1.000e+00, 32293749-2561-45aa-b788-df6153a6b965 = 1.000/1.000e+00, 58f72121-9cb2-46b9-b925-6e0c4e592981 = 1.000/1.000e+00, 472dd6db-3165-464a-92be-e733a6431524 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.2091506958428089, 1.2999158603115473, 1.4457195202954205, 1.4457195202954216
LBFGS Accumulation History: 3 points
Removed measurement 6dab09c0 to history. Total: 4
Removed measurement 56be778 to history. Total: 3
Adding measurement 4966863c to history. Total: 3
th(0)=1.2091506958428089;dx=-1.4590578576693316E-5
Adding measurement 361658f1 to history. Total: 4
New Minimum: 1.2091506958428089 > 1.021656801707239
WOLFE (weak): th(12164.510040883202)=1.021656801707239; dx=-1.662627332410265E-5 evalInputDelta=0.18749389413556994
Adding measurement 46b64be4 to history. Total: 5
New Minimum: 1.021656801707239 > 0.9242044736841745
WOLF (strong): th(24329.020081766404)=0.9242044736841745; dx=1.5738286304254947E-4 evalInputDelta=0.2849462221586344
Non-optimal measurement 0.9252749000925462 < 0.9242044736841745. Total: 6
WOLFE (weak): th(18246.765061324804)=0.9252749000925462; dx=-1.918937383432373E-5 evalInputDelta=0.28387579575026267
Adding measurement 11291c4c to history. Total: 6
New Minimum: 0.9242044736841745 > 0.8937126173822347
WOLFE (weak): th(21287.892571545606)=0.8937126173822347; dx=-2.178530829228237E-5 evalInputDelta=0.31543807846057415
Adding measurement 674dd0fc to history. Total: 7
New Minimum: 0.8937126173822347 > 0.8918920283207283
WOLFE (weak): th(22808.456326656007)=0.8918920283207283; dx=-2.212398486529769E-5 evalInputDelta=0.31725866752208054
Non-optimal measurement 0.8954342707030364 < 0.8918920283207283. Total: 8
WOLFE (weak): th(23568.738204211208)=0.8954342707030364; dx=-1.7081703692757505E-5 evalInputDelta=0.3137164251397725
Non-optimal measurement 0.9012219095415549 < 0.8918920283207283. Total: 8
END: th(23948.879142988808)=0.9012219095415549; dx=-4.671502057792514E-6 evalInputDelta=0.307928786301254
Fitness changed from 1.2091506958428089 to 0.8918920283207283
Iteration 106 complete. Error: 0.8918920283207283 Total: 0.0398; Orientation: 0.0308; Line Search: 0.0083
Non-optimal measurement 0.8918920283207283 < 0.8918920283207283. Total: 8
Accepted: LBFGS Orientation magnitude: 3.163e+02, gradient 1.848e-02, dot 0.372; [472dd6db-3165-464a-92be-e733a6431524 = 1.000/1.000e+00, 58f72121-9cb2-46b9-b925-6e0c4e592981 = 1.000/1.000e+00, 0cc7b127-97a6-4fa3-8b64-b517fdc17974 = 1.000/1.000e+00, 32293749-2561-45aa-b788-df6153a6b965 = 1.000/1.000e+00, f4566f37-3630-46d3-b16f-e2e3664dfea4 = 1.000/1.000e+00]
Non-optimal measurement 0.8918920283207283 < 0.8918920283207283. Total: 8
th(0)=0.8918920283207283;dx=-2.1739068903460703
Non-optimal measurement 0.8918920283207663 < 0.8918920283207283. Total: 8
Armijo: th(3.109917216915408E-13)=0.8918920283207663; dx=-2.173906890344749 evalInputDelta=-3.7969627442180354E-14
Non-optimal measurement 0.8918920283207473 < 0.8918920283207283. Total: 8
Armijo: th(1.554958608457704E-13)=0.8918920283207473; dx=-2.173906890345409 evalInputDelta=-1.8984813721090177E-14
Non-optimal measurement 0.8918920283207348 < 0.8918920283207283. Total: 8
Armijo: th(5.18319536152568E-14)=0.8918920283207348; dx=-2.17390689034585 evalInputDelta=-6.439293542825908E-15
Non-optimal measurement 0.8918920283207299 < 0.8918920283207283. Total: 8
Armijo: th(1.29579884038142E-14)=0.8918920283207299; dx=-2.1739068903460157 evalInputDelta=-1.5543122344752192E-15
Non-optimal measurement 0.8918920283207287 < 0.8918920283207283. Total: 8
Armijo: th(2.5915976807628397E-15)=0.8918920283207287; dx=-2.173906890346059 evalInputDelta=-3.3306690738754696E-16
Non-optimal measurement 0.8918920283207283 < 0.8918920283207283. Total: 8
MIN ALPHA (4.319329467938066E-16): th(0.0)=0.8918920283207283
Fitness changed from 0.8918920283207283 to 0.8918920283207283
Static Iteration Total: 0.0319; Orientation: 0.0246; Line Search: 0.0067
Iteration 107 failed. Error: 0.8918920283207283
Previous Error: 0.0 -> 0.8918920283207283
Optimization terminated 107
Final threshold in iteration 107: 0.8918920283207283 (> 0.0) after 10.019s (< 30.000s)

Returns

    0.8918920283207283

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

And regressed input:

TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [ 88.9577131411949, 80.13781435076352, 63.77290050624767, 275.87853816361894 ]

To produce the following output:

TrainingTester.java:633 executed in 0.00 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [ 0.17485649853074256, 0.15751998474871068, 0.12535288610637618, 0.5422706306141705 ]

TrainingTester.java:432 executed in 0.10 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging 
Plotting range=[1.0, -0.11532982951526913], [106.0, 0.30752564818259925]; valueStats=DoubleSummaryStatistics{count=124, sum=173.599221, min=0.766779, average=1.399994, max=2.030138}
Plotting 9 points for GD
Plotting 9 points for CjGD
Plotting 106 points for LBFGS

Returns

Result

TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging 
Plotting range=[0.0, -0.11532982951526913], [9.946, 0.30752564818259925]; valueStats=DoubleSummaryStatistics{count=124, sum=173.599221, min=0.766779, average=1.399994, max=2.030138}
Plotting 9 points for GD
Plotting 9 points for CjGD
Plotting 106 points for LBFGS

Returns

Result

Results

TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):

    return grid(inputLearning, modelLearning, completeLearning);

Returns

Result

TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):

    return new ComponentResult(null == inputLearning ? null : inputLearning.value,
        null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);

Returns

    {"input":{ "LBFGS": { "type": "NonConverged", "value": 0.8918920283207283 }, "CjGD": { "type": "NonConverged", "value": 0.7667789298456545 }, "GD": { "type": "NonConverged", "value": 0.9981266296865112 } }, "model":null, "complete":null}

LayerTests.java:425 executed in 0.00 seconds (0.000 gc):

    throwException(exceptions.addRef());

Results

detailsresult
{"input":{ "LBFGS": { "type": "NonConverged", "value": 0.8918920283207283 }, "CjGD": { "type": "NonConverged", "value": 0.7667789298456545 }, "GD": { "type": "NonConverged", "value": 0.9981266296865112 } }, "model":null, "complete":null}OK
 
  {
    "result": "OK",
    "performance": {
      "execution_time": "12.696",
      "gc_time": "1.542"
    },
    "created_on": 1586738555151,
    "file_name": "trainingTest",
    "report": {
      "simpleName": "Basic",
      "canonicalName": "com.simiacryptus.mindseye.layers.java.L1NormalizationLayerTest.Basic",
      "link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/test/java/com/simiacryptus/mindseye/layers/java/L1NormalizationLayerTest.java",
      "javaDoc": ""
    },
    "training_analysis": {
      "input": {
        "LBFGS": {
          "type": "NonConverged",
          "value": 0.8918920283207283
        },
        "CjGD": {
          "type": "NonConverged",
          "value": 0.7667789298456545
        },
        "GD": {
          "type": "NonConverged",
          "value": 0.9981266296865112
        }
      }
    },
    "archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/java/L1NormalizationLayer/Basic/trainingTest/202004134235",
    "id": "3801afb2-74dc-41cf-a52f-9b8775dd95c1",
    "report_type": "Components",
    "display_name": "Comparative Training",
    "target": {
      "simpleName": "L1NormalizationLayer",
      "canonicalName": "com.simiacryptus.mindseye.layers.java.L1NormalizationLayer",
      "link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/main/java/com/simiacryptus/mindseye/layers/java/L1NormalizationLayer.java",
      "javaDoc": ""
    }
  }