1. Test Modules
  2. Training Characteristics
    1. Input Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    2. Results
  3. Results

Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase

Test Modules

Using Seed 4761738593914081280

Training Characteristics

Input Learning

In this apply, we use a network to learn this target input, given it's pre-evaluated output:

TrainingTester.java:332 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
      try {
        return x.prettyPrint();
      } finally {
        x.freeRef();
      }
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [ -0.128, 0.08, -0.608, 0.496, 0.7 ]
    [ -0.608, 0.08, -0.128, 0.7, 0.496 ]
    [ 0.7, -0.608, 0.496, 0.08, -0.128 ]
    [ 0.496, -0.608, 0.7, -0.128, 0.08 ]
    [ 0.496, 0.08, 0.7, -0.128, -0.608 ]

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:480 executed in 0.40 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 4931965480314
BACKPROP_AGG_SIZE = 3
THREADS = 64
SINGLE_THREADED = false
Initialized CoreSettings = {
"backpropAggregationSize" : 3,
"jvmThreads" : 64,
"singleThreaded" : false
}
Reset training subject: 4931995733718
Constructing line search parameters: GD
th(0)=1.7293928934697014;dx=-0.014719462302748936
New Minimum: 1.7293928934697014 > 1.6987153469842688
WOLFE (weak): th(2.154434690031884)=1.6987153469842688; dx=-0.013751248052300376 evalInputDelta=0.030677546485432527
New Minimum: 1.6987153469842688 > 1.6701740290366272
END: th(4.308869380063768)=1.6701740290366272; dx=-0.012736453173605767 evalInputDelta=0.059218864433074225
Fitness changed from 1.7293928934697014 to 1.6701740290366272
Iteration 1 complete. Error: 1.6701740290366272 Total: 0.0694; Orientation: 0.0025; Line Search: 0.0258
th(0)=1.6701740290366272;dx=-0.011055792775498223
New Minimum: 1.6701740290366272 > 1.5835416063534191
END: th(9.283177667225559)=1.5835416063534191; dx=-0.007509104982059719 evalInputDelta=0.08663242268320803
Fitness changed from 1.6701740290366272 to 1.5835416063534191
Iteration 2 complete. Error: 1.5835416063534191 Total: 0.0173; Orientation: 0.0012; Line Search: 0.0117
th(0)=1.5835416063534191;dx=-0.005161626791558864
New Minimum: 1.5835416063534191 > 1.5206552975601917
END: th(20.000000000000004)=1.5206552975601917; dx=-9.889964658301926E-4 evalInputDelta=0.06288630879322743
Fitness changed from 1.5835416063534191 to 1.5206552975601917
Iteration 3 complete. Error: 1.5206552975601917 Total: 0.0188; Orientation: 0.0010; Line Search: 0.0137
th(0)=1.5206552975601917;dx=-2.983362634056112E-4
New Minimum: 1.5206552975601917 > 1.5169817808023485
WOLF (strong): th(43.088693800637685)=1.5169817808023485; dx=1.2416149800776607E-4 evalInputDelta=0.0036735167578432204
New Minimum: 1.5169817808023485 > 1.5165426270588207
END: th(21.544346900318843)=1.5165426270588207; dx=-8.433879644679211E-5 evalInputDelta=0.004112670501370985
Fitness changed from 1.5206552975601917 to 1.5165426270588207
Iteration 4 complete. Error: 1.5165426270588207 Total: 0.0351; Orientation: 0.0009; Line Search: 0.0306
th(0)=1.5165426270588207;dx=-3.846475836521881E-5
New Minimum: 1.5165426270588207 > 1.5157528464238772
WOLF (strong): th(46.4158883361278)=1.5157528464238772; dx=3.7087637444519304E-6 evalInputDelta=7.897806349435044E-4
END: th(23.2079441680639)=1.5159030903772066; dx=-1.6833955700508416E-5 evalInputDelta=6.395366816140946E-4
Fitness changed from 1.5165426270588207 to 1.5157528464238772
Iteration 5 complete. Error: 1.5157528464238772 Total: 0.0211; Orientation: 0.0011; Line Search: 0.0156
th(0)=1.5157528464238772;dx=-4.964043364827149E-6
Armijo: th(50.000000000000014)=1.515827495158081; dx=7.924300398990074E-6 evalInputDelta=-7.464873420381934E-5
New Minimum: 1.5157528464238772 > 1.5157096167446824
WOLF (strong): th(25.000000000000007)=1.5157096167446824; dx=1.499398444892604E-6 evalInputDelta=4.322967919478948E-5
END: th(8.333333333333336)=1.51572047662375; dx=-2.805393851625307E-6 evalInputDelta=3.2369800127218795E-5
Fitness changed from 1.5157528464238772 to 1.5157096167446824
Iteration 6 complete. Error: 1.5157096167446824 Total: 0.0257; Orientation: 0.0008; Line Search: 0.0213
th(0)=1.5157096167446824;dx=-8.658704491576975E-7
New Minimum: 1.5157096167446824 > 1.5157021398637922
WOLF (strong): th(17.95362241693237)=1.5157021398637922; dx=3.3046155217656104E-8 evalInputDelta=7.476880890200377E-6
END: th(8.976811208466184)=1.515703860951799; dx=-4.164768498821166E-7 evalInputDelta=5.755792883421762E-6
Fitness changed from 1.5157096167446824 to 1.5157021398637922
Iteration 7 complete. Error: 1.5157021398637922 Total: 0.0190; Orientation: 0.0009; Line Search: 0.0144
th(0)=1.5157021398637922;dx=-1.4607077923639362E-8
New Minimum: 1.5157021398637922 > 1.5157019589632998
END: th(19.339953473386583)=1.5157019589632998; dx=-4.0998609241182365E-9 evalInputDelta=1.809004923636337E-7
Fitness changed from 1.5157021398637922 to 1.5157019589632998
Iteration 8 complete. Error: 1.5157019589632998 Total: 0.0142; Orientation: 0.0006; Line Search: 0.0109
th(0)=1.5157019589632998;dx=-2.2226882421673165E-9
New Minimum: 1.5157019589632998 > 1.5157019211471041
WOLF (strong): th(41.66666666666668)=1.5157019211471041; dx=4.0765081302781447E-10 evalInputDelta=3.781619573040018E-8
END: th(20.83333333333334)=1.5157019263555196; dx=-9.076236883918316E-10 evalInputDelta=3.2607780253712804E-8
Fitness changed from 1.5157019589632998 to 1.5157019211471041
Iteration 9 complete. Error: 1.5157019211471041 Total: 0.0174; Orientation: 0.0008; Line Search: 0.0132
th(0)=1.5157019211471041;dx=-3.4058235995965896E-10
Armijo: th(44.884056042330926)=1.51570192189426; dx=3.7387603078111265E-10 evalInputDelta=-7.471558927107935E-10
New Minimum: 1.5157019211471041 > 1.515701917512208
WOLF (strong): th(22.442028021165463)=1.515701917512208; dx=1.6646110790383585E-11 evalInputDelta=3.6348961618415387E-9
END: th(7.480676007055155)=1.515701919044702; dx=-2.2150636442229668E-10 evalInputDelta=2.102402030246253E-9
Fitness changed from 1.5157019211471041 to 1.515701917512208
Iteration 10 complete. Error: 1.515701917512208 Total: 0.0384; Orientation: 0.0007; Line Search: 0.0354
Low gradient: 6.35455294527345E-6
th(0)=1.515701917512208;dx=-4.038034313428349E-11
New Minimum: 1.515701917512208 > 1.5157019170955173
END: th(16.116627894488822)=1.5157019170955173; dx=-1.132909671564427E-11 evalInputDelta=4.1669068195915315E-10
Fitness changed from 1.515701917512208 to 1.5157019170955173
Iteration 11 complete. Error: 1.5157019170955173 Total: 0.0111; Orientation: 0.0007; Line Search: 0.0073
Low gradient: 2.482057924143093E-6
th(0)=1.5157019170955173;dx=-6.1606115388015194E-12
New Minimum: 1.5157019170955173 > 1.5157019169743278
END: th(34.722222222222236)=1.5157019169743278; dx=-8.199190039870891E-13 evalInputDelta=1.211895028774279E-10
Fitness changed from 1.5157019170955173 to 1.5157019169743278
Iteration 12 complete. Error: 1.5157019169743278 Total: 0.0102; Orientation: 0.0008; Line Search: 0.0071
Low gradient: 6.276973840881435E-7
th(0)=1.5157019169743278;dx=-3.9400400599109835E-13
New Minimum: 1.5157019169743278 > 1.5157019169716863
WOLF (strong): th(74.80676007055155)=1.5157019169716863; dx=3.233823326948964E-13 evalInputDelta=2.6414426201881724E-12
New Minimum: 1.5157019169716863 > 1.5157019169662989
END: th(37.403380035275774)=1.5157019169662989; dx=-3.5309854712646716E-14 evalInputDelta=8.028910869484207E-12
Fitness changed from 1.5157019169743278 to 1.5157019169662989
Iteration 13 complete. Error: 1.5157019169662989 Total: 0.0109; Orientation: 0.0008; Line Search: 0.0083
Low gradient: 2.91659030100659E-7
th(0)=1.5157019169662989;dx=-8.506498983925711E-14
Armijo: th(80.58313947244412)=1.515701916972866; dx=2.480633505227663E-13 evalInputDelta=-6.567191235262726E-12
New Minimum: 1.5157019169662989 > 1.515701916966227
WOLF (strong): th(40.29156973622206)=1.515701916966227; dx=8.149915991336252E-14 evalInputDelta=7.194245199571014E-14
New Minimum: 1.515701916966227 > 1.5157019169655293
END: th(13.430523245407352)=1.5157019169655293; dx=-2.9543611127614255E-14 evalInputDelta=7.696066006701585E-13
Fitness changed from 1.5157019169662989 to 1.5157019169655293
Iteration 14 complete. Error: 1.5157019169655293 Total: 0.0155; Orientation: 0.0007; Line Search: 0.0122
Low gradient: 1.2281777940697294E-7
th(0)=1.5157019169655293;dx=-1.508420693845987E-14
New Minimum: 1.5157019169655293 > 1.515701916965259
END: th(28.935185185185198)=1.515701916965259; dx=-3.577327142435913E-15 evalInputDelta=2.702282841937631E-13
Fitness changed from 1.5157019169655293 to 1.515701916965259
Iteration 15 complete. Error: 1.515701916965259 Total: 0.0068; Orientation: 0.0004; Line Search: 0.0046
Low gradient: 5.987094861567026E-8
th(0)=1.515701916965259;dx=-3.584530488140229E-15
Armijo: th(62.33896672545963)=1.5157019169652728; dx=4.026437844958446E-15 evalInputDelta=-1.3766765505351941E-14
New Minimum: 1.515701916965259 > 1.5157019169652066
WOLF (strong): th(31.169483362729814)=1.5157019169652066; dx=2.209540638707424E-16 evalInputDelta=5.240252676230739E-14
END: th(10.389827787576605)=1.5157019169652284; dx=-2.316035551388772E-15 evalInputDelta=3.064215547965432E-14
Fitness changed from 1.515701916965259 to 1.5157019169652066
Iteration 16 complete. Error: 1.5157019169652066 Total: 0.0159; Orientation: 0.0007; Line Search: 0.0133
Low gradient: 3.5843332796954796E-8
th(0)=1.5157019169652066;dx=-1.2847445059932558E-15
New Minimum: 1.5157019169652066 > 1.515701916965194
WOLF (strong): th(22.384205409012257)=1.515701916965194; dx=1.250834472801131E-16 evalInputDelta=1.2656542480726785E-14
END: th(11.192102704506128)=1.5157019169651962; dx=-5.798305331823645E-16 evalInputDelta=1.0436096431476471E-14
Fitness changed from 1.5157019169652066 to 1.515701916965194
Iteration 17 complete. Error: 1.515701916965194 Total: 0.0160; Orientation: 0.0007; Line Search: 0.0134
Low gradient: 1.4515564776465834E-8
th(0)=1.515701916965194;dx=-2.1070162077977563E-16
New Minimum: 1.515701916965194 > 1.5157019169651913
END: th(24.11265432098767)=1.5157019169651913; dx=-9.783085438679791E-18 evalInputDelta=2.6645352591003757E-15
Fitness changed from 1.515701916965194 to 1.5157019169651913
Iteration 18 complete. Error: 1.5157019169651913 Total: 0.0076; Orientation: 0.0004; Line Search: 0.0051
Low gradient: 7.000886894701339E-9
th(0)=1.5157019169651913;dx=-4.9012417312400954E-17
New Minimum: 1.5157019169651913 > 1.515701916965191
WOLF (strong): th(51.949138937883035)=1.515701916965191; dx=4.246006615880052E-17 evalInputDelta=2.220446049250313E-16
New Minimum: 1.515701916965191 > 1.5157019169651904
END: th(25.974569468941517)=1.5157019169651904; dx=-3.276175320473871E-18 evalInputDelta=8.881784197001252E-16
Fitness changed from 1.5157019169651913 to 1.5157019169651904
Iteration 19 complete. Error: 1.5157019169651904 Total: 0.0115; Orientation: 0.0004; Line Search: 0.0090
Low gradient: 3.58735252446522E-9
th(0)=1.5157019169651904;dx=-1.2869098134786987E-17
WOLF (strong): th(55.960513522530654)=1.5157019169651904; dx=1.4625122691891857E-17 evalInputDelta=0.0
WOLF (strong): th(27.980256761265327)=1.5157019169651904; dx=8.78012342839207E-19 evalInputDelta=0.0
END: th(9.326752253755108)=1.5157019169651904; dx=-8.28672797050812E-18 evalInputDelta=0.0
Fitness changed from 1.5157019169651904 to 1.5157019169651904
Static Iteration Total: 0.0131; Orientation: 0.0006; Line Search: 0.0107
Iteration 20 failed. Error: 1.5157019169651904
Previous Error: 0.0 -> 1.5157019169651904
Optimization terminated 20
Final threshold in iteration 20: 1.5157019169651904 (> 0.0) after 0.395s (< 30.000s)

Returns

    1.5157019169651904

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

And regressed input:

TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [ -0.12799999947347038, 0.07999999968886898, -0.6080000038040222, 0.49599999339514494, 0.7000000101934793 ]

To produce the following output:

TrainingTester.java:633 executed in 0.00 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [ 0.14275289453544854, 0.17575924359603826, 0.088333119888258, 0.2664309520137715, 0.32672378996648366 ]

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:452 executed in 0.44 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 4932374406692
Reset training subject: 4932376712443
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=1.7293928934697014}, derivative=-0.014719462302748936}
New Minimum: 1.7293928934697014 > 1.7293928934682299
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=1.7293928934682299}, derivative=-0.014719462302705082}, evalInputDelta = -1.4714895968381825E-12
New Minimum: 1.7293928934682299 > 1.7293928934593983
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=1.7293928934593983}, derivative=-0.014719462302441961}, evalInputDelta = -1.0303091713126378E-11
New Minimum: 1.7293928934593983 > 1.7293928933975764
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=1.7293928933975764}, derivative=-0.014719462300600106}, evalInputDelta = -7.212497266095852E-11
New Minimum: 1.7293928933975764 > 1.7293928929648241
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=1.7293928929648241}, derivative=-0.014719462287707138}, evalInputDelta = -5.048772511173638E-10
New Minimum: 1.7293928929648241 > 1.7293928899355588
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=1.7293928899355588}, derivative=-0.014719462197456349}, evalInputDelta = -3.534142534178386E-9
New Minimum: 1.7293928899355588 > 1.7293928687307019
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=1.7293928687307019}, derivative=-0.014719461565700807}, evalInputDelta = -2.4738999515605542E-8
New Minimum: 1.7293928687307019 > 1.72939272029673
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=1.72939272029673}, derivative=-0.01471945714341144}, evalInputDelta = -1.7317297129615383E-7
New Minimum: 1.72939272029673 > 1.7293916812601746
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=1.7293916812601746}, derivative=-0.01471942618735713}, evalInputDelta = -1.2122095267574906E-6
New Minimum: 1.7293916812601746 > 1.7293844080654712
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=1.7293844080654712}, derivative=-0.0147192094935686}, evalInputDelta = -8.485404230151872E-6
New Minimum: 1.7293844080654712 > 1.7293334987007036
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=1.7293334987007036}, derivative=-0.014717692568038811}, evalInputDelta = -5.939476899774476E-5
New Minimum: 1.7293334987007036 > 1.7289772800887733
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=1.7289772800887733}, derivative=-0.014707070707811094}, evalInputDelta = -4.1561338092810374E-4
New Minimum: 1.7289772800887733 > 1.7264909608254473
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=1.7264909608254473}, derivative=-0.014632551984190025}, evalInputDelta = -0.0029019326442540905
New Minimum: 1.7264909608254473 > 1.709443801063366
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=1.709443801063366}, derivative=-0.014102802467011254}, evalInputDelta = -0.019949092406335467
New Minimum: 1.709443801063366 > 1.6088736744281178
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=1.6088736744281178}, derivative=-0.01000635234675889}, evalInputDelta = -0.12051921904158358
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=1.9780801518403925}, derivative=0.019816299192228402}, evalInputDelta = 0.24868725837069117
F(5.217100560376924) = LineSearchPoint{point=PointSample{avg=1.6588062973491666}, derivative=-0.01229488258394301}, evalInputDelta = -0.07058659612053475
New Minimum: 1.6088736744281178 > 1.5503851667055897
F(36.51970392263847) = LineSearchPoint{point=PointSample{avg=1.5503851667055897}, derivative=0.005894982151248583}, evalInputDelta = -0.1790077267641117
1.5503851667055897 <= 1.7293928934697014
New Minimum: 1.5503851667055897 > 1.52154659692786
F(26.076395432164485) = LineSearchPoint{point=PointSample{avg=1.52154659692786}, derivative=-4.23233432591449E-4}, evalInputDelta = -0.20784629654184128
Left bracket at 26.076395432164485
New Minimum: 1.52154659692786 > 1.5214008247299344
F(26.77595328063986) = LineSearchPoint{point=PointSample{avg=1.5214008247299344}, derivative=6.491374219075584E-6}, evalInputDelta = -0.20799206873976694
Right bracket at 26.77595328063986
Converged to right
Fitness changed from 1.7293928934697014 to 1.5214008247299344
Iteration 1 complete. Error: 1.5214008247299344 Total: 0.0650; Orientation: 0.0005; Line Search: 0.0579
F(0.0) = LineSearchPoint{point=PointSample{avg=1.5214008247299344}, derivative=-4.7675491650705663E-4}
New Minimum: 1.5214008247299344 > 1.516954787233998
F(26.77595328063986) = LineSearchPoint{point=PointSample{avg=1.516954787233998}, derivative=1.435358495636238E-4}, evalInputDelta = -0.004446037495936528
1.516954787233998 <= 1.5214008247299344
New Minimum: 1.516954787233998 > 1.5165071126154692
F(20.5799732463754) = LineSearchPoint{point=PointSample{avg=1.5165071126154692}, derivative=8.171885870382357E-7}, evalInputDelta = -0.00489371211446521
Right bracket at 20.5799732463754
Converged to right
Fitness changed from 1.5214008247299344 to 1.5165071126154692
Iteration 2 complete. Error: 1.5165071126154692 Total: 0.0101; Orientation: 0.0003; Line Search: 0.0082
F(0.0) = LineSearchPoint{point=PointSample{avg=1.5165071126154692}, derivative=-4.6976658915100134E-5}
New Minimum: 1.5165071126154692 > 1.515896777407436
F(20.5799732463754) = LineSearchPoint{point=PointSample{avg=1.515896777407436}, derivative=-1.2365810638796221E-5}, evalInputDelta = -6.103352080333302E-4
F(144.0598127246278) = LineSearchPoint{point=PointSample{avg=1.5269702495737736}, derivative=1.8984693972239059E-4}, evalInputDelta = 0.010463136958304364
F(11.0815240557406) = LineSearchPoint{point=PointSample{avg=1.5160899647736397}, derivative=-2.8318394763268328E-5}, evalInputDelta = -4.171478418295571E-4
F(77.5706683901842) = LineSearchPoint{point=PointSample{avg=1.5178999991875022}, derivative=8.234828537191668E-5}, evalInputDelta = 0.0013928865720329586
F(5.96697449155263) = LineSearchPoint{point=PointSample{avg=1.5162568028991013}, derivative=-3.6923953326670676E-5}, evalInputDelta = -2.5030971636796906E-4
F(41.76882144086841) = LineSearchPoint{point=PointSample{avg=1.5160105461054454}, derivative=2.3066612631691632E-5}, evalInputDelta = -4.965665100238326E-4
1.5160105461054454 <= 1.5165071126154692
New Minimum: 1.515896777407436 > 1.5158511681614262
F(28.013535558552455) = LineSearchPoint{point=PointSample{avg=1.5158511681614262}, derivative=9.028538539763881E-8}, evalInputDelta = -6.559444540430448E-4
Right bracket at 28.013535558552455
Converged to right
Fitness changed from 1.5165071126154692 to 1.5158511681614262
Iteration 3 complete. Error: 1.5158511681614262 Total: 0.0467; Orientation: 0.0003; Line Search: 0.0447
F(0.0) = LineSearchPoint{point=PointSample{avg=1.5158511681614262}, derivative=-1.049335165782475E-5}
New Minimum: 1.5158511681614262 > 1.5157439556334422
F(28.013535558552455) = LineSearchPoint{point=PointSample{avg=1.5157439556334422}, derivative=2.825896130671607E-6}, evalInputDelta = -1.0721252798395753E-4
1.5157439556334422 <= 1.5158511681614262
New Minimum: 1.5157439556334422 > 1.5157355359879245
F(22.07000608913924) = LineSearchPoint{point=PointSample{avg=1.5157355359879245}, derivative=6.6872443639006774E-9}, evalInputDelta = -1.1563217350163413E-4
Right bracket at 22.07000608913924
Converged to right
Fitness changed from 1.5158511681614262 to 1.5157355359879245
Iteration 4 complete. Error: 1.5157355359879245 Total: 0.0128; Orientation: 0.0006; Line Search: 0.0102
F(0.0) = LineSearchPoint{point=PointSample{avg=1.5157355359879245}, derivative=-1.7855178217378727E-6}
New Minimum: 1.5157355359879245 > 1.51571117531287
F(22.07000608913924) = LineSearchPoint{point=PointSample{avg=1.51571117531287}, derivative=-4.225785210372841E-7}, evalInputDelta = -2.4360675054513337E-5
F(154.4900426239747) = LineSearchPoint{point=PointSample{avg=1.516193528516508}, derivative=7.688404523320418E-6}, evalInputDelta = 4.5799252858347295E-4
F(11.883849432613438) = LineSearchPoint{point=PointSample{avg=1.5157186810463135}, derivative=-1.0512439561115021E-6}, evalInputDelta = -1.685494161107748E-5
F(83.18694602829407) = LineSearchPoint{point=PointSample{avg=1.51580030961914}, de

...skipping 13451 bytes...

plete. Error: 1.5157019169662724 Total: 0.0086; Orientation: 0.0005; Line Search: 0.0065
Low gradient: 2.377865924621188E-7
F(0.0) = LineSearchPoint{point=PointSample{avg=1.5157019169662724}, derivative=-5.654246355474577E-14}
New Minimum: 1.5157019169662724 > 1.5157019169654948
F(22.22639565209883) = LineSearchPoint{point=PointSample{avg=1.5157019169654948}, derivative=-1.343895419624714E-14}, evalInputDelta = -7.776002064474596E-13
F(155.5847695646918) = LineSearchPoint{point=PointSample{avg=1.5157019169809471}, derivative=2.451817144768018E-13}, evalInputDelta = 1.4674705894890394E-11
F(11.968059197283985) = LineSearchPoint{point=PointSample{avg=1.5157019169657344}, derivative=-3.3332879300254484E-14}, evalInputDelta = -5.380140777333509E-13
F(83.77641438098789) = LineSearchPoint{point=PointSample{avg=1.5157019169683408}, derivative=1.0592451388665641E-13}, evalInputDelta = 2.0683454948766666E-12
F(6.444339567768299) = LineSearchPoint{point=PointSample{avg=1.5157019169659482}, derivative=-4.404499444498977E-14}, evalInputDelta = -3.241851231905457E-13
F(45.1103769743781) = LineSearchPoint{point=PointSample{avg=1.515701916965695}, derivative=3.0939787643411204E-14}, evalInputDelta = -5.775380174100064E-13
1.515701916965695 <= 1.5157019169662724
New Minimum: 1.5157019169654948 > 1.5157019169654482
F(29.15622096003346) = LineSearchPoint{point=PointSample{avg=1.5157019169654482}, derivative=8.686693614168952E-21}, evalInputDelta = -8.242295734817162E-13
Right bracket at 29.15622096003346
Converged to right
Fitness changed from 1.5157019169662724 to 1.5157019169654482
Iteration 17 complete. Error: 1.5157019169654482 Total: 0.0177; Orientation: 0.0004; Line Search: 0.0158
Low gradient: 1.3296005518677612E-7
F(0.0) = LineSearchPoint{point=PointSample{avg=1.5157019169654482}, derivative=-1.7678376275270554E-14}
New Minimum: 1.5157019169654482 > 1.5157019169652708
F(29.15622096003346) = LineSearchPoint{point=PointSample{avg=1.5157019169652708}, derivative=5.5118149355322806E-15}, evalInputDelta = -1.7741363933510002E-13
1.5157019169652708 <= 1.5157019169654482
New Minimum: 1.5157019169652708 > 1.5157019169652517
F(22.226407717427243) = LineSearchPoint{point=PointSample{avg=1.5157019169652517}, derivative=6.121071989279216E-22}, evalInputDelta = -1.965094753586527E-13
Right bracket at 22.226407717427243
Converged to right
Fitness changed from 1.5157019169654482 to 1.5157019169652517
Iteration 18 complete. Error: 1.5157019169652517 Total: 0.0119; Orientation: 0.0006; Line Search: 0.0097
Low gradient: 5.667524749698644E-8
F(0.0) = LineSearchPoint{point=PointSample{avg=1.5157019169652517}, derivative=-3.2120836788446683E-15}
New Minimum: 1.5157019169652517 > 1.5157019169652073
F(22.226407717427243) = LineSearchPoint{point=PointSample{avg=1.5157019169652073}, derivative=-7.634460651359845E-16}, evalInputDelta = -4.440892098500626E-14
F(155.5848540219907) = LineSearchPoint{point=PointSample{avg=1.5157019169660852}, derivative=1.392837437301318E-14}, evalInputDelta = 8.335554468885675E-13
F(11.968065693999286) = LineSearchPoint{point=PointSample{avg=1.5157019169652213}, derivative=-1.893586471136115E-15}, evalInputDelta = -3.042011087472929E-14
F(83.77645985799501) = LineSearchPoint{point=PointSample{avg=1.5157019169653692}, derivative=6.017395255070824E-15}, evalInputDelta = 1.1746159600534156E-13
F(6.444343065999616) = LineSearchPoint{point=PointSample{avg=1.5157019169652333}, derivative=-2.5021236351839985E-15}, evalInputDelta = -1.84297022087776E-14
F(45.11040146199731) = LineSearchPoint{point=PointSample{avg=1.5157019169652188}, derivative=1.7576361882927E-15}, evalInputDelta = -3.2862601528904634E-14
1.5157019169652188 <= 1.5157019169652517
New Minimum: 1.5157019169652073 > 1.5157019169652048
F(29.156247868287966) = LineSearchPoint{point=PointSample{avg=1.5157019169652048}, derivative=1.1742733809947286E-22}, evalInputDelta = -4.6851411639181606E-14
Right bracket at 29.156247868287966
Converged to right
Fitness changed from 1.5157019169652517 to 1.5157019169652048
Iteration 19 complete. Error: 1.5157019169652048 Total: 0.0264; Orientation: 0.0005; Line Search: 0.0201
Low gradient: 3.1690431197143805E-8
F(0.0) = LineSearchPoint{point=PointSample{avg=1.5157019169652048}, derivative=-1.004283429460905E-15}
New Minimum: 1.5157019169652048 > 1.5157019169651949
F(29.156247868287966) = LineSearchPoint{point=PointSample{avg=1.5157019169651949}, derivative=3.131194173729421E-16}, evalInputDelta = -9.992007221626409E-15
1.5157019169651949 <= 1.5157019169652048
New Minimum: 1.5157019169651949 > 1.5157019169651937
F(22.226410600029183) = LineSearchPoint{point=PointSample{avg=1.5157019169651937}, derivative=8.276692859993136E-24}, evalInputDelta = -1.1102230246251565E-14
Right bracket at 22.226410600029183
Converged to right
Fitness changed from 1.5157019169652048 to 1.5157019169651937
Iteration 20 complete. Error: 1.5157019169651937 Total: 0.0092; Orientation: 0.0005; Line Search: 0.0072
Low gradient: 1.3508298830606101E-8
F(0.0) = LineSearchPoint{point=PointSample{avg=1.5157019169651937}, derivative=-1.8247413729695418E-16}
New Minimum: 1.5157019169651937 > 1.5157019169651913
F(22.226410600029183) = LineSearchPoint{point=PointSample{avg=1.5157019169651913}, derivative=-4.33703583491537E-17}, evalInputDelta = -2.4424906541753444E-15
F(155.5848742002043) = LineSearchPoint{point=PointSample{avg=1.515701916965241}, derivative=7.912522443353174E-16}, evalInputDelta = 4.729550084903167E-14
F(11.96806724616956) = LineSearchPoint{point=PointSample{avg=1.5157019169651922}, derivative=-1.0757210211240676E-16}, evalInputDelta = -1.5543122344752192E-15
F(83.77647072318692) = LineSearchPoint{point=PointSample{avg=1.5157019169652004}, derivative=3.418400887042115E-16}, evalInputDelta = 6.661338147750939E-15
F(6.4443439017836095) = LineSearchPoint{point=PointSample{avg=1.5157019169651926}, derivative=-1.4214227206556078E-16}, evalInputDelta = -1.1102230246251565E-15
F(45.110407312485265) = LineSearchPoint{point=PointSample{avg=1.5157019169651917}, derivative=9.984891348901587E-17}, evalInputDelta = -1.9984014443252818E-15
1.5157019169651917 <= 1.5157019169651937
New Minimum: 1.5157019169651913 > 1.515701916965191
F(29.156254278720848) = LineSearchPoint{point=PointSample{avg=1.515701916965191}, derivative=1.5276679820341199E-24}, evalInputDelta = -2.6645352591003757E-15
Right bracket at 29.156254278720848
Converged to right
Fitness changed from 1.5157019169651937 to 1.515701916965191
Iteration 21 complete. Error: 1.515701916965191 Total: 0.0221; Orientation: 0.0003; Line Search: 0.0206
Low gradient: 7.553280214548002E-9
F(0.0) = LineSearchPoint{point=PointSample{avg=1.515701916965191}, derivative=-5.705204199948231E-17}
New Minimum: 1.515701916965191 > 1.5157019169651904
F(29.156254278720848) = LineSearchPoint{point=PointSample{avg=1.5157019169651904}, derivative=1.7787923070029075E-17}, evalInputDelta = -6.661338147750939E-16
1.5157019169651904 <= 1.515701916965191
F(22.226411277880462) = LineSearchPoint{point=PointSample{avg=1.5157019169651904}, derivative=5.840020481524748E-27}, evalInputDelta = -6.661338147750939E-16
Right bracket at 22.226411277880462
Converged to right
Fitness changed from 1.515701916965191 to 1.5157019169651904
Iteration 22 complete. Error: 1.5157019169651904 Total: 0.0097; Orientation: 0.0005; Line Search: 0.0074
Low gradient: 4.835678374721147E-9
F(0.0) = LineSearchPoint{point=PointSample{avg=1.5157019169651904}, derivative=-2.3383785343745754E-17}
F(22.226411277880462) = LineSearchPoint{point=PointSample{avg=1.5157019169651904}, derivative=4.239676382233081E-18}, evalInputDelta = 0.0
1.5157019169651904 <= 1.5157019169651904
F(18.815079566764535) = LineSearchPoint{point=PointSample{avg=1.5157019169651904}, derivative=-6.901950962432558E-26}, evalInputDelta = 0.0
Left bracket at 18.815079566764535
Converged to left
Fitness changed from 1.5157019169651904 to 1.5157019169651904
Static Iteration Total: 0.0084; Orientation: 0.0005; Line Search: 0.0067
Iteration 23 failed. Error: 1.5157019169651904
Previous Error: 0.0 -> 1.5157019169651904
Optimization terminated 23
Final threshold in iteration 23: 1.5157019169651904 (> 0.0) after 0.437s (< 30.000s)

Returns

    1.5157019169651904

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

And regressed input:

TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [ -0.1279999973244999, 0.08000000427230793, -0.6079999984613588, 0.4960000255845401, 0.6999999659290104 ]

To produce the following output:

TrainingTester.java:633 executed in 0.00 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [ 0.1427528954563027, 0.17575924515768632, 0.08833312074017613, 0.2664309617361331, 0.3267237769097018 ]

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:509 executed in 0.86 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 4932822707768
Reset training subject: 4932824206185
Adding measurement 17e40b87 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 1.7293928934697014 < 1.7293928934697014. Total: 1
th(0)=1.7293928934697014;dx=-0.014719462302748936
Adding measurement 151a7c9e to history. Total: 1
New Minimum: 1.7293928934697014 > 1.6987153469842688
WOLFE (weak): th(2.154434690031884)=1.6987153469842688; dx=-0.013751248052300376 evalInputDelta=0.030677546485432527
Adding measurement 7bdd626c to history. Total: 2
New Minimum: 1.6987153469842688 > 1.6701740290366272
END: th(4.308869380063768)=1.6701740290366272; dx=-0.012736453173605767 evalInputDelta=0.059218864433074225
Fitness changed from 1.7293928934697014 to 1.6701740290366272
Iteration 1 complete. Error: 1.6701740290366272 Total: 0.0136; Orientation: 0.0023; Line Search: 0.0072
Non-optimal measurement 1.6701740290366272 < 1.6701740290366272. Total: 3
LBFGS Accumulation History: 3 points
Non-optimal measurement 1.6701740290366272 < 1.6701740290366272. Total: 3
th(0)=1.6701740290366272;dx=-0.011055792775498223
Adding measurement 500ae50e to history. Total: 3
New Minimum: 1.6701740290366272 > 1.5835416063534191
END: th(9.283177667225559)=1.5835416063534191; dx=-0.007509104982059719 evalInputDelta=0.08663242268320803
Fitness changed from 1.6701740290366272 to 1.5835416063534191
Iteration 2 complete. Error: 1.5835416063534191 Total: 0.0063; Orientation: 0.0011; Line Search: 0.0038
Non-optimal measurement 1.5835416063534191 < 1.5835416063534191. Total: 4
Rejected: LBFGS Orientation magnitude: 2.983e+00, gradient 7.184e-02, dot -0.973; [f80a9222-1686-4101-a3d8-0fcdce10c9a1 = 1.000/1.000e+00, 3c62b19d-55e8-4c8f-b616-8c574cd4bbdf = 1.000/1.000e+00, 0d82e393-c438-4ac2-ae76-5b1e5b731fcf = 1.000/1.000e+00, 06b731a1-8689-4109-8463-96cf9f4f43d6 = 1.000/1.000e+00, 6c41b4cd-bba8-42d8-8835-0e0ccfc15add = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.5835416063534191, 1.6701740290366272, 1.6987153469842688, 1.7293928934697014
LBFGS Accumulation History: 3 points
Removed measurement 500ae50e to history. Total: 3
Adding measurement 316857a7 to history. Total: 3
th(0)=1.5835416063534191;dx=-0.005161626791558864
Adding measurement 2a3c2108 to history. Total: 4
New Minimum: 1.5835416063534191 > 1.5206552975601917
END: th(20.000000000000004)=1.5206552975601917; dx=-9.889964658301926E-4 evalInputDelta=0.06288630879322743
Fitness changed from 1.5835416063534191 to 1.5206552975601917
Iteration 3 complete. Error: 1.5206552975601917 Total: 0.0227; Orientation: 0.0180; Line Search: 0.0037
Non-optimal measurement 1.5206552975601917 < 1.5206552975601917. Total: 5
Rejected: LBFGS Orientation magnitude: 6.374e-01, gradient 1.727e-02, dot -0.965; [0d82e393-c438-4ac2-ae76-5b1e5b731fcf = 1.000/1.000e+00, 3c62b19d-55e8-4c8f-b616-8c574cd4bbdf = 1.000/1.000e+00, 6c41b4cd-bba8-42d8-8835-0e0ccfc15add = 1.000/1.000e+00, f80a9222-1686-4101-a3d8-0fcdce10c9a1 = 1.000/1.000e+00, 06b731a1-8689-4109-8463-96cf9f4f43d6 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.5206552975601917, 1.5835416063534191, 1.6701740290366272, 1.6987153469842688, 1.7293928934697014
Rejected: LBFGS Orientation magnitude: 6.102e-01, gradient 1.727e-02, dot -0.974; [06b731a1-8689-4109-8463-96cf9f4f43d6 = 1.000/1.000e+00, f80a9222-1686-4101-a3d8-0fcdce10c9a1 = 1.000/1.000e+00, 6c41b4cd-bba8-42d8-8835-0e0ccfc15add = 1.000/1.000e+00, 3c62b19d-55e8-4c8f-b616-8c574cd4bbdf = 1.000/1.000e+00, 0d82e393-c438-4ac2-ae76-5b1e5b731fcf = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.5206552975601917, 1.5835416063534191, 1.6701740290366272, 1.6987153469842688
LBFGS Accumulation History: 3 points
Removed measurement 2a3c2108 to history. Total: 4
Removed measurement 316857a7 to history. Total: 3
Adding measurement 407f70ca to history. Total: 3
th(0)=1.5206552975601917;dx=-2.983362634056112E-4
Adding measurement 6283de80 to history. Total: 4
New Minimum: 1.5206552975601917 > 1.5169817808023485
WOLF (strong): th(43.088693800637685)=1.5169817808023485; dx=1.2416149800776607E-4 evalInputDelta=0.0036735167578432204
Adding measurement 7f580373 to history. Total: 5
New Minimum: 1.5169817808023485 > 1.5165426270588207
END: th(21.544346900318843)=1.5165426270588207; dx=-8.433879644679211E-5 evalInputDelta=0.004112670501370985
Fitness changed from 1.5206552975601917 to 1.5165426270588207
Iteration 4 complete. Error: 1.5165426270588207 Total: 0.0419; Orientation: 0.0342; Line Search: 0.0066
Non-optimal measurement 1.5165426270588207 < 1.5165426270588207. Total: 6
Rejected: LBFGS Orientation magnitude: 2.133e-01, gradient 6.202e-03, dot -0.984; [f80a9222-1686-4101-a3d8-0fcdce10c9a1 = 1.000/1.000e+00, 6c41b4cd-bba8-42d8-8835-0e0ccfc15add = 1.000/1.000e+00, 3c62b19d-55e8-4c8f-b616-8c574cd4bbdf = 1.000/1.000e+00, 0d82e393-c438-4ac2-ae76-5b1e5b731fcf = 1.000/1.000e+00, 06b731a1-8689-4109-8463-96cf9f4f43d6 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.5165426270588207, 1.5169817808023485, 1.5206552975601917, 1.6701740290366272, 1.6987153469842688, 1.7293928934697014
Rejected: LBFGS Orientation magnitude: 2.129e-01, gradient 6.202e-03, dot -0.986; [f80a9222-1686-4101-a3d8-0fcdce10c9a1 = 1.000/1.000e+00, 06b731a1-8689-4109-8463-96cf9f4f43d6 = 1.000/1.000e+00, 0d82e393-c438-4ac2-ae76-5b1e5b731fcf = 1.000/1.000e+00, 3c62b19d-55e8-4c8f-b616-8c574cd4bbdf = 1.000/1.000e+00, 6c41b4cd-bba8-42d8-8835-0e0ccfc15add = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.5165426270588207, 1.5169817808023485, 1.5206552975601917, 1.6701740290366272, 1.6987153469842688
Rejected: LBFGS Orientation magnitude: 2.187e-01, gradient 6.202e-03, dot -0.994; [3c62b19d-55e8-4c8f-b616-8c574cd4bbdf = 1.000/1.000e+00, 6c41b4cd-bba8-42d8-8835-0e0ccfc15add = 1.000/1.000e+00, 0d82e393-c438-4ac2-ae76-5b1e5b731fcf = 1.000/1.000e+00, 06b731a1-8689-4109-8463-96cf9f4f43d6 = 1.000/1.000e+00, f80a9222-1686-4101-a3d8-0fcdce10c9a1 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.5165426270588207, 1.5169817808023485, 1.5206552975601917, 1.6701740290366272
LBFGS Accumulation History: 3 points
Removed measurement 7f580373 to history. Total: 5
Removed measurement 6283de80 to history. Total: 4
Removed measurement 407f70ca to history. Total: 3
Adding measurement 459bf940 to history. Total: 3
th(0)=1.5165426270588207;dx=-3.846475836521881E-5
Adding measurement 61813ba5 to history. Total: 4
New Minimum: 1.5165426270588207 > 1.5157528464238772
WOLF (strong): th(46.4158883361278)=1.5157528464238772; dx=3.7087637444519304E-6 evalInputDelta=7.897806349435044E-4
Non-optimal measurement 1.5159030903772066 < 1.5157528464238772. Total: 5
END: th(23.2079441680639)=1.5159030903772066; dx=-1.6833955700508416E-5 evalInputDelta=6.395366816140946E-4
Fitness changed from 1.5165426270588207 to 1.5157528464238772
Iteration 5 complete. Error: 1.5157528464238772 Total: 0.0564; Orientation: 0.0491; Line Search: 0.0061
Non-optimal measurement 1.5157528464238772 < 1.5157528464238772. Total: 5
Rejected: LBFGS Orientation magnitude: 5.536e-02, gradient 2.228e-03, dot -0.974; [0d82e393-c438-4ac2-ae76-5b1e5b731fcf = 1.000/1.000e+00, 6c41b4cd-bba8-42d8-8835-0e0ccfc15add = 1.000/1.000e+00, 06b731a1-8689-4109-8463-96cf9f4f43d6 = 1.000/1.000e+00, 3c62b19d-55e8-4c8f-b616-8c574cd4bbdf = 1.000/1.000e+00, f80a9222-1686-4101-a3d8-0fcdce10c9a1 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.5157528464238772, 1.5165426270588207, 1.6701740290366272, 1.6987153469842688, 1.7293928934697014
Rejected: LBFGS Orientation magnitude: 5.458e-02, gradient 2.228e-03, dot -0.978; [6c41b4cd-bba8-42d8-8835-0e0ccfc15add = 1.000/1.000e+00, f80a9222-1686-4101-a3d8-0fcdce10c9a1 = 1.000/1.000e+00, 3c62b19d-55e8-4c8f-b616-8c574cd4bbdf = 1.000/1.000e+00, 0d82e393-c438-4ac2-ae76-5b1e5b731fcf = 1.000/1.000e+00, 06b731a1-8689-4109-8463-96cf9f4f43d6 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.5157528464238772, 1.5165426270588207, 1.6701740290366272, 1.6987153469842688
LBFGS Accumulation History: 3 points
Removed measurement 61813ba5 to hi

...skipping 20832 bytes...

169652066. Total: 5
Rejected: LBFGS Orientation magnitude: 1.036e-06, gradient 3.584e-08, dot -0.896; [06b731a1-8689-4109-8463-96cf9f4f43d6 = 1.000/1.000e+00, 3c62b19d-55e8-4c8f-b616-8c574cd4bbdf = 1.000/1.000e+00, f80a9222-1686-4101-a3d8-0fcdce10c9a1 = 1.000/1.000e+00, 0d82e393-c438-4ac2-ae76-5b1e5b731fcf = 1.000/1.000e+00, 6c41b4cd-bba8-42d8-8835-0e0ccfc15add = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.5157019169652066, 1.515701916965259, 1.6701740290366272, 1.6987153469842688, 1.7293928934697014
Rejected: LBFGS Orientation magnitude: 1.013e-06, gradient 3.584e-08, dot -0.900; [f80a9222-1686-4101-a3d8-0fcdce10c9a1 = 1.000/1.000e+00, 3c62b19d-55e8-4c8f-b616-8c574cd4bbdf = 1.000/1.000e+00, 06b731a1-8689-4109-8463-96cf9f4f43d6 = 1.000/1.000e+00, 6c41b4cd-bba8-42d8-8835-0e0ccfc15add = 1.000/1.000e+00, 0d82e393-c438-4ac2-ae76-5b1e5b731fcf = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.5157019169652066, 1.515701916965259, 1.6701740290366272, 1.6987153469842688
LBFGS Accumulation History: 3 points
Removed measurement 27c65c59 to history. Total: 4
Removed measurement 1e1c8140 to history. Total: 3
Adding measurement 3cd2a434 to history. Total: 3
th(0)=1.5157019169652066;dx=-1.2847445059932558E-15
Adding measurement 235d95f4 to history. Total: 4
New Minimum: 1.5157019169652066 > 1.515701916965194
WOLF (strong): th(22.384205409012257)=1.515701916965194; dx=1.2508344728011313E-16 evalInputDelta=1.2656542480726785E-14
Non-optimal measurement 1.5157019169651962 < 1.515701916965194. Total: 5
END: th(11.192102704506128)=1.5157019169651962; dx=-5.798305331823645E-16 evalInputDelta=1.0436096431476471E-14
Fitness changed from 1.5157019169652066 to 1.515701916965194
Iteration 17 complete. Error: 1.515701916965194 Total: 0.0367; Orientation: 0.0290; Line Search: 0.0065
Non-optimal measurement 1.515701916965194 < 1.515701916965194. Total: 5
Rejected: LBFGS Orientation magnitude: 4.655e-07, gradient 1.452e-08, dot -0.953; [3c62b19d-55e8-4c8f-b616-8c574cd4bbdf = 1.000/1.000e+00, 6c41b4cd-bba8-42d8-8835-0e0ccfc15add = 1.000/1.000e+00, f80a9222-1686-4101-a3d8-0fcdce10c9a1 = 1.000/1.000e+00, 06b731a1-8689-4109-8463-96cf9f4f43d6 = 1.000/1.000e+00, 0d82e393-c438-4ac2-ae76-5b1e5b731fcf = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.515701916965194, 1.5157019169652066, 1.6701740290366272, 1.6987153469842688, 1.7293928934697014
Rejected: LBFGS Orientation magnitude: 4.521e-07, gradient 1.452e-08, dot -0.957; [0d82e393-c438-4ac2-ae76-5b1e5b731fcf = 1.000/1.000e+00, 06b731a1-8689-4109-8463-96cf9f4f43d6 = 1.000/1.000e+00, f80a9222-1686-4101-a3d8-0fcdce10c9a1 = 1.000/1.000e+00, 3c62b19d-55e8-4c8f-b616-8c574cd4bbdf = 1.000/1.000e+00, 6c41b4cd-bba8-42d8-8835-0e0ccfc15add = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.515701916965194, 1.5157019169652066, 1.6701740290366272, 1.6987153469842688
LBFGS Accumulation History: 3 points
Removed measurement 235d95f4 to history. Total: 4
Removed measurement 3cd2a434 to history. Total: 3
Adding measurement 67296344 to history. Total: 3
th(0)=1.515701916965194;dx=-2.1070162077977563E-16
Adding measurement 572e9770 to history. Total: 4
New Minimum: 1.515701916965194 > 1.5157019169651913
END: th(24.11265432098767)=1.5157019169651913; dx=-9.783085438679791E-18 evalInputDelta=2.6645352591003757E-15
Fitness changed from 1.515701916965194 to 1.5157019169651913
Iteration 18 complete. Error: 1.5157019169651913 Total: 0.0412; Orientation: 0.0322; Line Search: 0.0075
Non-optimal measurement 1.5157019169651913 < 1.5157019169651913. Total: 5
Rejected: LBFGS Orientation magnitude: 2.181e-07, gradient 7.001e-09, dot -0.913; [6c41b4cd-bba8-42d8-8835-0e0ccfc15add = 1.000/1.000e+00, 06b731a1-8689-4109-8463-96cf9f4f43d6 = 1.000/1.000e+00, 3c62b19d-55e8-4c8f-b616-8c574cd4bbdf = 1.000/1.000e+00, 0d82e393-c438-4ac2-ae76-5b1e5b731fcf = 1.000/1.000e+00, f80a9222-1686-4101-a3d8-0fcdce10c9a1 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.5157019169651913, 1.515701916965194, 1.6701740290366272, 1.6987153469842688, 1.7293928934697014
Rejected: LBFGS Orientation magnitude: 2.155e-07, gradient 7.001e-09, dot -0.914; [3c62b19d-55e8-4c8f-b616-8c574cd4bbdf = 1.000/1.000e+00, 0d82e393-c438-4ac2-ae76-5b1e5b731fcf = 1.000/1.000e+00, 6c41b4cd-bba8-42d8-8835-0e0ccfc15add = 1.000/1.000e+00, f80a9222-1686-4101-a3d8-0fcdce10c9a1 = 1.000/1.000e+00, 06b731a1-8689-4109-8463-96cf9f4f43d6 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.5157019169651913, 1.515701916965194, 1.6701740290366272, 1.6987153469842688
LBFGS Accumulation History: 3 points
Removed measurement 572e9770 to history. Total: 4
Removed measurement 67296344 to history. Total: 3
Adding measurement 4977f9bb to history. Total: 3
th(0)=1.5157019169651913;dx=-4.9012417312400954E-17
Adding measurement 7ed09447 to history. Total: 4
New Minimum: 1.5157019169651913 > 1.515701916965191
WOLF (strong): th(51.949138937883035)=1.515701916965191; dx=4.246006615880052E-17 evalInputDelta=2.220446049250313E-16
Adding measurement 353fcfcc to history. Total: 5
New Minimum: 1.515701916965191 > 1.5157019169651904
END: th(25.974569468941517)=1.5157019169651904; dx=-3.276175320473871E-18 evalInputDelta=8.881784197001252E-16
Fitness changed from 1.5157019169651913 to 1.5157019169651904
Iteration 19 complete. Error: 1.5157019169651904 Total: 0.0472; Orientation: 0.0398; Line Search: 0.0061
Non-optimal measurement 1.5157019169651904 < 1.5157019169651904. Total: 6
Rejected: LBFGS Orientation magnitude: 1.326e-07, gradient 3.587e-09, dot -0.951; [06b731a1-8689-4109-8463-96cf9f4f43d6 = 1.000/1.000e+00, 3c62b19d-55e8-4c8f-b616-8c574cd4bbdf = 1.000/1.000e+00, 0d82e393-c438-4ac2-ae76-5b1e5b731fcf = 1.000/1.000e+00, 6c41b4cd-bba8-42d8-8835-0e0ccfc15add = 1.000/1.000e+00, f80a9222-1686-4101-a3d8-0fcdce10c9a1 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.5157019169651904, 1.515701916965191, 1.5157019169651913, 1.6701740290366272, 1.6987153469842688, 1.7293928934697014
Rejected: LBFGS Orientation magnitude: 1.297e-07, gradient 3.587e-09, dot -0.951; [06b731a1-8689-4109-8463-96cf9f4f43d6 = 1.000/1.000e+00, 6c41b4cd-bba8-42d8-8835-0e0ccfc15add = 1.000/1.000e+00, 3c62b19d-55e8-4c8f-b616-8c574cd4bbdf = 1.000/1.000e+00, f80a9222-1686-4101-a3d8-0fcdce10c9a1 = 1.000/1.000e+00, 0d82e393-c438-4ac2-ae76-5b1e5b731fcf = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.5157019169651904, 1.515701916965191, 1.5157019169651913, 1.6701740290366272, 1.6987153469842688
Rejected: LBFGS Orientation magnitude: 1.232e-07, gradient 3.587e-09, dot -0.935; [3c62b19d-55e8-4c8f-b616-8c574cd4bbdf = 1.000/1.000e+00, 6c41b4cd-bba8-42d8-8835-0e0ccfc15add = 1.000/1.000e+00, 06b731a1-8689-4109-8463-96cf9f4f43d6 = 1.000/1.000e+00, 0d82e393-c438-4ac2-ae76-5b1e5b731fcf = 1.000/1.000e+00, f80a9222-1686-4101-a3d8-0fcdce10c9a1 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 1.5157019169651904, 1.515701916965191, 1.5157019169651913, 1.6701740290366272
LBFGS Accumulation History: 3 points
Removed measurement 353fcfcc to history. Total: 5
Removed measurement 7ed09447 to history. Total: 4
Removed measurement 4977f9bb to history. Total: 3
Adding measurement 3877ef49 to history. Total: 3
th(0)=1.5157019169651904;dx=-1.2869098134786987E-17
Non-optimal measurement 1.5157019169651904 < 1.5157019169651904. Total: 4
WOLF (strong): th(55.960513522530654)=1.5157019169651904; dx=1.4625122691891857E-17 evalInputDelta=0.0
Non-optimal measurement 1.5157019169651904 < 1.5157019169651904. Total: 4
WOLF (strong): th(27.980256761265327)=1.5157019169651904; dx=8.780123428392073E-19 evalInputDelta=0.0
Non-optimal measurement 1.5157019169651904 < 1.5157019169651904. Total: 4
END: th(9.326752253755108)=1.5157019169651904; dx=-8.28672797050812E-18 evalInputDelta=0.0
Fitness changed from 1.5157019169651904 to 1.5157019169651904
Static Iteration Total: 0.0568; Orientation: 0.0475; Line Search: 0.0080
Iteration 20 failed. Error: 1.5157019169651904
Previous Error: 0.0 -> 1.5157019169651904
Optimization terminated 20
Final threshold in iteration 20: 1.5157019169651904 (> 0.0) after 0.858s (< 30.000s)

Returns

    1.5157019169651904

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

And regressed input:

TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [ -0.12799999947347038, 0.07999999968886898, -0.6080000038040222, 0.49599999339514494, 0.7000000101934793 ]

To produce the following output:

TrainingTester.java:633 executed in 0.00 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [ 0.14275289453544854, 0.17575924359603826, 0.088333119888258, 0.2664309520137715, 0.32672378996648366 ]

TrainingTester.java:432 executed in 0.11 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging
Plotting range=[1.0, 0.1806137998805067], [22.0, 0.22276172618502113]; valueStats=DoubleSummaryStatistics{count=60, sum=91.405142, min=1.515702, average=1.523419, max=1.670174}
Plotting 19 points for GD
Plotting 22 points for CjGD
Plotting 19 points for LBFGS

Returns

Result

TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging
Plotting range=[0.0, 0.1806137998805067], [0.787, 0.22276172618502113]; valueStats=DoubleSummaryStatistics{count=60, sum=91.405142, min=1.515702, average=1.523419, max=1.670174}
Plotting 19 points for GD
Plotting 22 points for CjGD
Plotting 19 points for LBFGS

Returns

Result

Results

TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):

    return grid(inputLearning, modelLearning, completeLearning);

Returns

Result

TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):

    return new ComponentResult(null == inputLearning ? null : inputLearning.value,
        null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);

Returns

    {"input":{ "LBFGS": { "type": "NonConverged", "value": 1.5157019169651904 }, "CjGD": { "type": "NonConverged", "value": 1.5157019169651904 }, "GD": { "type": "NonConverged", "value": 1.5157019169651904 } }, "model":null, "complete":null}

LayerTests.java:425 executed in 0.00 seconds (0.000 gc):

    throwException(exceptions.addRef());

Results

detailsresult
{"input":{ "LBFGS": { "type": "NonConverged", "value": 1.5157019169651904 }, "CjGD": { "type": "NonConverged", "value": 1.5157019169651904 }, "GD": { "type": "NonConverged", "value": 1.5157019169651904 } }, "model":null, "complete":null}OK
  {
    "result": "OK",
    "performance": {
      "execution_time": "2.635",
      "gc_time": "0.461"
    },
    "created_on": 1586739519770,
    "file_name": "trainingTest",
    "report": {
      "simpleName": "SubBatchLayerTest",
      "canonicalName": "com.simiacryptus.mindseye.layers.java.SubBatchLayerTest",
      "link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/test/java/com/simiacryptus/mindseye/layers/java/SubBatchLayerTest.java",
      "javaDoc": ""
    },
    "training_analysis": {
      "input": {
        "LBFGS": {
          "type": "NonConverged",
          "value": 1.5157019169651904
        },
        "CjGD": {
          "type": "NonConverged",
          "value": 1.5157019169651904
        },
        "GD": {
          "type": "NonConverged",
          "value": 1.5157019169651904
        }
      }
    },
    "archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/java/SubBatchLayer/SubBatchLayerTest/trainingTest/202004135839",
    "id": "b4aedc54-8e85-4c4a-80f4-334ed3dc1bd6",
    "report_type": "Components",
    "display_name": "Comparative Training",
    "target": {
      "simpleName": "SubBatchLayer",
      "canonicalName": "com.simiacryptus.mindseye.layers.java.SubBatchLayer",
      "link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/main/java/com/simiacryptus/mindseye/layers/java/SubBatchLayer.java",
      "javaDoc": ""
    }
  }