1. Test Modules
  2. Training Characteristics
    1. Input Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    2. Results
  3. Results

Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase

Test Modules

Using Seed 1798522284047326208

Training Characteristics

Input Learning

In this apply, we use a network to learn this target input, given it's pre-evaluated output:

TrainingTester.java:332 executed in 0.02 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
      try {
        return x.prettyPrint();
      } finally {
        x.freeRef();
      }
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ -0.528 ], [ 0.44 ], [ -0.3 ], [ 0.092 ], [ -0.188 ], [ 1.872 ], [ 0.144 ], [ -1.536 ], ... ],
    	[ [ -1.752 ], [ -1.796 ], [ -0.336 ], [ 1.444 ], [ -1.124 ], [ -0.444 ], [ -1.868 ], [ -1.048 ], ... ],
    	[ [ -0.884 ], [ 1.456 ], [ -1.712 ], [ 1.768 ], [ 1.1 ], [ 1.72 ], [ 1.288 ], [ -1.684 ], ... ],
    	[ [ 1.54 ], [ 0.332 ], [ -0.236 ], [ 1.468 ], [ -1.26 ], [ -1.668 ], [ -1.388 ], [ -0.82 ], ... ],
    	[ [ -1.784 ], [ 0.936 ], [ -0.268 ], [ 1.256 ], [ 0.348 ], [ -0.604 ], [ 0.464 ], [ -0.352 ], ... ],
    	[ [ 1.456 ], [ 0.044 ], [ -1.784 ], [ 1.908 ], [ 1.564 ], [ -1.656 ], [ -1.144 ], [ 1.736 ], ... ],
    	[ [ 0.804 ], [ -0.684 ], [ 1.196 ], [ 0.036 ], [ -0.78 ], [ 1.34 ], [ 0.34 ], [ 0.372 ], ... ],
    	[ [ -0.492 ], [ -1.344 ], [ 0.268 ], [ -1.08 ], [ 0.016 ], [ 1.076 ], [ 1.332 ], [ -0.228 ], ... ],
    	...
    ]
    [
    	[ [ -1.1 ], [ -1.48 ], [ -0.9 ], [ -1.32 ], [ 0.276 ], [ -1.356 ], [ -0.52 ], [ 0.388 ], ... ],
    	[ [ -1.272 ], [ 1.9 ], [ 1.0 ], [ 0.076 ], [ -1.376 ], [ 0.432 ], [ 1.216 ], [ 1.644 ], ... ],
    	[ [ 1.048 ], [ -0.96 ], [ 0.208 ], [ -0.872 ], [ -0.84 ], [ -0.028 ], [ 1.48 ], [ 0.616 ], ... ],
    	[ [ -0.804 ], [ -0.16 ], [ 0.68 ], [ 0.508 ], [ 1.752 ], [ -0.236 ], [ 1.136 ], [ -1.028 ], ... ],
    	[ [ -0.408 ], [ 0.2 ], [ -1.948 ], [ 0.8 ], [ -0.104 ], [ 1.864 ], [ 1.076 ], [ 1.7 ], ... ],
    	[ [ 0.164 ], [ 1.576 ], [ 0.752 ], [ -1.508 ], [ 1.168 ], [ 1.952 ], [ 0.424 ], [ -0.16 ], ... ],
    	[ [ -0.82 ], [ 1.616 ], [ 0.872 ], [ 0.696 ], [ 0.164 ], [ -1.66 ], [ 0.06 ], [ 1.488 ], ... ],
    	[ [ 0.48 ], [ -1.2 ], [ 0.244 ], [ -1.084 ], [ 0.612 ], [ 0.54 ], [ 1.224 ], [ -1.592 ], ... ],
    	...
    ]
    [
    	[ [ -1.236 ], [ -0.748 ], [ 0.3 ], [ -1.164 ], [ 1.424 ], [ 0.516 ], [ 1.408 ], [ 0.116 ], ... ],
    	[ [ 1.064 ], [ 1.864 ], [ -1.312 ], [ 0.06 ], [ -1.976 ], [ 0.612 ], [ 1.044 ], [ -0.204 ], ... ],
    	[ [ -0.776 ], [ -1.868 ], [ 0.108 ], [ 1.564 ], [ -1.176 ], [ -1.672 ], [ 1.552 ], [ -0.212 ], ... ],
    	[ [ -1.88 ], [ -1.196 ], [ 1.304 ], [ 0.876 ], [ -1.94 ], [ 1.792 ], [ -1.088 ], [ -1.104 ], ... ],
    	[ [ -0.524 ], [ -1.124 ], [ 1.936 ], [ 0.116 ], [ 0.328 ], [ 1.54 ], [ -1.408 ], [ 1.18 ], ... ],
    	[ [ 1.212 ], [ -1.1 ], [ -1.476 ], [ 0.44 ], [ 1.82 ], [ -1.072 ], [ 1.0 ], [ 0.448 ], ... ],
    	[ [ -1.448 ], [ -0.272 ], [ 0.02 ], [ 1.412 ], [ 0.84 ], [ -0.6 ], [ -1.524 ], [ 0.204 ], ... ],
    	[ [ 1.948 ], [ -1.636 ], [ 0.732 ], [ 1.824 ], [ 0.96 ], [ 0.992 ], [ 0.668 ], [ -0.9 ], ... ],
    	...
    ]
    [
    	[ [ 0.72 ], [ -0.668 ], [ -1.192 ], [ 1.744 ], [ -1.772 ], [ -0.036 ], [ -0.084 ], [ 0.396 ], ... ],
    	[ [ 1.256 ], [ 1.64 ], [ -0.3 ], [ 0.648 ], [ 1.384 ], [ -0.676 ], [ 0.824 ], [ 0.644 ], ... ],
    	[ [ -0.064 ], [ 1.592 ], [ 1.808 ], [ 1.688 ], [ -1.956 ], [ -1.4 ], [ 0.652 ], [ -2.0 ], ... ],
    	[ [ 0.128 ], [ 0.752 ], [ 1.056 ], [ 0.352 ], [ 0.372 ], [ -0.048 ], [ -0.968 ], [ 1.744 ], ... ],
    	[ [ 1.156 ], [ -1.92 ], [ 1.576 ], [ 1.488 ], [ 0.872 ], [ -0.152 ], [ 1.54 ], [ -1.872 ], ... ],
    	[ [ -0.192 ], [ 1.096 ], [ 0.884 ], [ -0.268 ], [ -1.968 ], [ -1.552 ], [ -1.232 ], [ 0.036 ], ... ],
    	[ [ -0.248 ], [ -0.556 ], [ 0.728 ], [ -1.628 ], [ -1.584 ], [ -0.532 ], [ 1.948 ], [ -1.616 ], ... ],
    	[ [ -0.428 ], [ 0.64 ], [ -0.652 ], [ 0.048 ], [ -1.468 ], [ -1.324 ], [ 1.168 ], [ 1.116 ], ... ],
    	...
    ]
    [
    	[ [ -0.72 ], [ 0.116 ], [ 0.424 ], [ -1.336 ], [ 1.008 ], [ 0.16 ], [ -0.3 ], [ -0.06 ], ... ],
    	[ [ 0.608 ], [ -0.072 ], [ 0.188 ], [ -1.496 ], [ -1.468 ], [ 1.14 ], [ 0.076 ], [ -1.68 ], ... ],
    	[ [ -0.66 ], [ -0.092 ], [ -1.856 ], [ 1.716 ], [ 0.16 ], [ 1.0 ], [ -0.856 ], [ -0.52 ], ... ],
    	[ [ -1.184 ], [ 1.196 ], [ 1.056 ], [ 0.672 ], [ 0.476 ], [ 1.764 ], [ -1.604 ], [ 0.696 ], ... ],
    	[ [ -0.332 ], [ 0.78 ], [ -0.812 ], [ 1.856 ], [ 1.496 ], [ 0.0 ], [ 0.024 ], [ -1.84 ], ... ],
    	[ [ -1.808 ], [ 0.008 ], [ -0.048 ], [ 1.312 ], [ -0.012 ], [ 1.208 ], [ 0.332 ], [ 1.88 ], ... ],
    	[ [ 0.64 ], [ -0.524 ], [ 1.828 ], [ 0.512 ], [ -1.184 ], [ -0.56 ], [ 1.536 ], [ 1.132 ], ... ],
    	[ [ -0.804 ], [ 0.788 ], [ -0.22 ], [ 0.58 ], [ 0.568 ], [ -1.028 ], [ -1.148 ], [ 0.604 ], ... ],
    	...
    ]

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:480 executed in 30.13 seconds (0.849 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 10775714093850
Reset training subject: 10776012879039
Constructing line search parameters: GD
th(0)=0.9435411534569775;dx=-4.169551618244071E-7
New Minimum: 0.9435411534569775 > 0.9435402551545264
WOLFE (weak): th(2.154434690031884)=0.9435402551545264; dx=-4.1695496337097924E-7 evalInputDelta=8.983024510911264E-7
New Minimum: 0.9435402551545264 > 0.9435393568525029
WOLFE (weak): th(4.308869380063768)=0.9435393568525029; dx=-4.169547649175514E-7 evalInputDelta=1.796604474635366E-6
New Minimum: 0.9435393568525029 > 0.9435357636486845
WOLFE (weak): th(12.926608140191302)=0.9435357636486845; dx=-4.1695397110383903E-7 evalInputDelta=5.38980829301039E-6
New Minimum: 0.9435357636486845 > 0.9435195943161576
WOLFE (weak): th(51.70643256076521)=0.9435195943161576; dx=-4.169503989421203E-7 evalInputDelta=2.1559140819915612E-5
New Minimum: 0.9435195943161576 > 0.9434333602155945
WOLFE (weak): th(258.53216280382605)=0.9434333602155945; dx=-4.169313474125812E-7 evalInputDelta=1.0779324138299184E-4
New Minimum: 0.9434333602155945 > 0.9428944863605574
WOLFE (weak): th(1551.1929768229563)=0.9428944863605574; dx=-4.168122753387498E-7 evalInputDelta=6.466670964201393E-4
New Minimum: 0.9428944863605574 > 0.9390191383192983
WOLFE (weak): th(10858.350837760694)=0.9390191383192983; dx=-4.159549556847592E-7 evalInputDelta=0.004522015137679247
New Minimum: 0.9390191383192983 > 0.9076691301549058
WOLFE (weak): th(86866.80670208555)=0.9076691301549058; dx=-4.0895346484413216E-7 evalInputDelta=0.03587202330207173
New Minimum: 0.9076691301549058 > 0.6457167927012458
END: th(781801.26031877)=0.6457167927012458; dx=-3.449363124072154E-7 evalInputDelta=0.2978243607557317
Fitness changed from 0.9435411534569775 to 0.6457167927012458
Iteration 1 complete. Error: 0.6457167927012458 Total: 5.3141; Orientation: 0.2155; Line Search: 4.5789
th(0)=0.6457167927012458;dx=-2.8535696539382136E-7
New Minimum: 0.6457167927012458 > 0.2545162161702924
END: th(1684339.7559414052)=0.2545162161702924; dx=-1.7915620239784022E-7 evalInputDelta=0.3912005765309534
Fitness changed from 0.6457167927012458 to 0.2545162161702924
Iteration 2 complete. Error: 0.2545162161702924 Total: 0.5769; Orientation: 0.1131; Line Search: 0.3664
th(0)=0.2545162161702924;dx=-1.1247997693403811E-7
New Minimum: 0.2545162161702924 > 0.00999381973576718
END: th(3628800.0)=0.00999381973576718; dx=-2.228811823100917E-8 evalInputDelta=0.24452239643452522
Fitness changed from 0.2545162161702924 to 0.00999381973576718
Iteration 3 complete. Error: 0.00999381973576718 Total: 0.6978; Orientation: 0.1355; Line Search: 0.4556
th(0)=0.00999381973576718;dx=-4.416432420550618E-9
New Minimum: 0.00999381973576718 > 0.005288326662799393
WOLF (strong): th(7818012.6031877)=0.005288326662799393; dx=3.2125815275913333E-9 evalInputDelta=0.004705493072967788
New Minimum: 0.005288326662799393 > 1.8560164139987565E-4
END: th(3909006.30159385)=1.8560164139987565E-4; dx=-6.018549066904682E-10 evalInputDelta=0.009808218094367304
Fitness changed from 0.00999381973576718 to 1.8560164139987565E-4
Iteration 4 complete. Error: 1.8560164139987565E-4 Total: 0.7580; Orientation: 0.1140; Line Search: 0.5461
Low gradient: 9.056408986466268E-6
th(0)=1.8560164139987565E-4;dx=-8.201854373014699E-11
New Minimum: 1.8560164139987565E-4 > 1.3752572126491953E-4
WOLF (strong): th(8421698.779707026)=1.3752572126491953E-4; dx=7.06011024228172E-11 evalInputDelta=4.807592013495612E-5
New Minimum: 1.3752572126491953E-4 > 8.990927332481069E-7
END: th(4210849.389853513)=8.990927332481069E-7; dx=-5.7085059177049116E-12 evalInputDelta=1.8470254866662753E-4
Fitness changed from 1.8560164139987565E-4 to 8.990927332481069E-7
Iteration 5 complete. Error: 8.990927332481069E-7 Total: 0.7679; Orientation: 0.1148; Line Search: 0.5558
Low gradient: 6.303279169397894E-7
th(0)=8.990927332481069E-7;dx=-3.9731328287365404E-13
Armijo: th(9072000.0)=9.071654448848538E-7; dx=3.990928687601591E-13 evalInputDelta=-8.072711636746939E-9
New Minimum: 8.990927332481069E-7 > 4.513030604880461E-12
WOLF (strong): th(4536000.0)=4.513030604880461E-12; dx=8.898772778358669E-16 evalInputDelta=8.99088220217502E-7
END: th(1512000.0)=3.9870226240523786E-7; dx=-2.6457887744262333E-13 evalInputDelta=5.00390470842869E-7
Fitness changed from 8.990927332481069E-7 to 4.513030604880461E-12
Iteration 6 complete. Error: 4.513030604880461E-12 Total: 0.9595; Orientation: 0.1145; Line Search: 0.7484
Low gradient: 1.4122090247456334E-9
th(0)=4.513030604880461E-12;dx=-1.9943343295730136E-18
New Minimum: 4.513030604880461E-12 > 3.544405424171839E-13
END: th(3257505.2513282085)=3.544405424171839E-13; dx=-5.589018075064839E-19 evalInputDelta=4.158590062463277E-12
Fitness changed from 4.513030604880461E-12 to 3.544405424171839E-13
Iteration 7 complete. Error: 3.544405424171839E-13 Total: 0.7415; Orientation: 0.2396; Line Search: 0.3945
Low gradient: 3.957642271475613E-10
th(0)=3.544405424171839E-13;dx=-1.566293234897065E-19
New Minimum: 3.544405424171839E-13 > 1.0747791594133442E-13
WOLF (strong): th(7018082.316422523)=1.0747791594133442E-13; dx=8.62503751431266E-20 evalInputDelta=2.469626264758495E-13
New Minimum: 1.0747791594133442E-13 > 1.7890514430023486E-14
END: th(3509041.1582112615)=1.7890514430023486E-14; dx=-3.5189474185207006E-20 evalInputDelta=3.365500279871604E-13
Fitness changed from 3.544405424171839E-13 to 1.7890514430023486E-14
Iteration 8 complete. Error: 1.7890514430023486E-14 Total: 0.8317; Orientation: 0.1231; Line Search: 0.6006
Zero gradient: 8.891524909080217E-11
th(0)=1.7890514430023486E-14;dx=-7.905921520879396E-21
New Minimum: 1.7890514430023486E-14 > 8.040735496967625E-15
WOLF (strong): th(7560000.000000002)=8.040735496967625E-15; dx=5.300159898583653E-21 evalInputDelta=9.849778933055862E-15
New Minimum: 8.040735496967625E-15 > 4.858780221254697E-16
END: th(3780000.000000001)=4.858780221254697E-16; dx=-1.3028808113074685E-21 evalInputDelta=1.7404636407898017E-14
Fitness changed from 1.7890514430023486E-14 to 4.858780221254697E-16
Iteration 9 complete. Error: 4.858780221254697E-16 Total: 0.8804; Orientation: 0.1259; Line Search: 0.6459
Zero gradient: 1.4653064314183466E-11
th(0)=4.858780221254697E-16;dx=-2.1471229379559696E-22
New Minimum: 4.858780221254697E-16 > 3.104862665156669E-16
WOLF (strong): th(8143763.128320523)=3.104862665156669E-16; dx=1.7163840943965945E-22 evalInputDelta=1.7539175560980285E-16
New Minimum: 3.104862665156669E-16 > 4.8885678726669235E-18
END: th(4071881.5641602613)=4.8885678726669235E-18; dx=-2.1536942183843378E-23 evalInputDelta=4.809894542528028E-16
Fitness changed from 4.858780221254697E-16 to 4.8885678726669235E-18
Iteration 10 complete. Error: 4.8885678726669235E-18 Total: 0.7615; Orientation: 0.1141; Line Search: 0.5489
Zero gradient: 1.4697911932574806E-12
th(0)=4.8885678726669235E-18;dx=-2.160286151777249E-24
New Minimum: 4.8885678726669235E-18 > 4.30422008552607E-18
WOLF (strong): th(8772602.895528154)=4.30422008552607E-18; dx=2.027065078365144E-24 evalInputDelta=5.843477871408535E-19
New Minimum: 4.30422008552607E-18 > 4.647788688268871E-21
END: th(4386301.447764077)=4.647788688268871E-21; dx=-6.661053675757843E-26 evalInputDelta=4.883920083978655E-18
Fitness changed from 4.8885678726669235E-18 to 4.647788688268871E-21
Iteration 11 complete. Error: 4.647788688268871E-21 Total: 0.8057; Orientation: 0.1117; Line Search: 0.5992
Zero gradient: 4.531979753132592E-14
th(0)=4.647788688268871E-21;dx=-2.053884048280374E-27
Armijo: th(9450000.000000004)=5.5018341046595006E-21; dx=2.2346344012166876E-27 evalInputDelta=-8.540454163906292E-22
New Minimum: 4.647788688268871E-21 > 8.998984931994433E-24
WOLF (strong): th(4725000.000000002)=8.998984931994433E-24; dx=9.037517429111916E-29 evalInputDelta=4.638789703336877E-21
END: th(1575000.0000000007)=1.975789348005625E-21; dx=-1.3391309684117673E-27 evalInputDelta=2.6719993402632464E-21
Fitness changed from 4.647788688268871E-21 to 8.998984931994433E-24
Iteration 12 complete. Error: 8.998984931994433E-24 Total: 0.8832; Orientation: 0.1146; Line Search: 0.6708
Zero gradient: 1.9941671970579116E-15
th(0)=8.998984931994433E-24;dx=-3.976702809821807E-30
New Minimum: 8.998984931994433E-24 > 5.63583666

...skipping 4343 bytes...

205E-47 evalInputDelta=1.2170459206018094E-38
Fitness changed from 1.2229650894447762E-38 to 5.919168842966824E-41
Iteration 20 complete. Error: 5.919168842966824E-41 Total: 0.7338; Orientation: 0.1121; Line Search: 0.5268
Zero gradient: 5.114614125947601E-24
th(0)=5.919168842966824E-41;dx=-2.6159277657342744E-47
New Minimum: 5.919168842966824E-41 > 5.693435443534651E-41
WOLF (strong): th(8203125.000000008)=5.693435443534651E-41; dx=2.5608918131108114E-47 evalInputDelta=2.2573339943217365E-42
New Minimum: 5.693435443534651E-41 > 5.832722636536448E-44
END: th(4101562.500000004)=5.832722636536448E-44; dx=-2.7517976311731625E-49 evalInputDelta=5.913336120330288E-41
Fitness changed from 5.919168842966824E-41 to 5.832722636536448E-44
Iteration 21 complete. Error: 5.832722636536448E-44 Total: 0.7494; Orientation: 0.1143; Line Search: 0.5385
Zero gradient: 1.6055290656577977E-25
th(0)=5.832722636536448E-44;dx=-2.5777235806720003E-50
New Minimum: 5.832722636536448E-44 > 5.293105393244898E-44
WOLF (strong): th(8836548.533333907)=5.293105393244898E-44; dx=2.4555905462058934E-50 evalInputDelta=5.3961724329154957E-45
New Minimum: 5.293105393244898E-44 > 3.2734441817121825E-47
END: th(4418274.266666953)=3.2734441817121825E-47; dx=-6.1066517233053655E-52 evalInputDelta=5.829449192354735E-44
Fitness changed from 5.832722636536448E-44 to 3.2734441817121825E-47
Iteration 22 complete. Error: 3.2734441817121825E-47 Total: 0.9413; Orientation: 0.1149; Line Search: 0.7290
Zero gradient: 3.803513654113402E-27
th(0)=3.2734441817121825E-47;dx=-1.4466716117027085E-53
Armijo: th(9518883.350182466)=3.985361240367155E-47; dx=1.5962515639859404E-53 evalInputDelta=-7.119170586549725E-48
New Minimum: 3.2734441817121825E-47 > 8.74886692128943E-50
WOLF (strong): th(4759441.675091233)=8.74886692128943E-50; dx=7.478997614161599E-55 evalInputDelta=3.264695314790893E-47
END: th(1586480.5583637443)=1.3806226669778007E-47; dx=-9.395177490879336E-54 evalInputDelta=1.8928215147343818E-47
Fitness changed from 3.2734441817121825E-47 to 8.74886692128943E-50
Iteration 23 complete. Error: 8.74886692128943E-50 Total: 0.9794; Orientation: 0.1132; Line Search: 0.7689
Zero gradient: 1.9663390996567753E-28
th(0)=8.74886692128943E-50;dx=-3.8664894548390176E-56
New Minimum: 8.74886692128943E-50 > 5.2398736793350074E-51
END: th(3417968.7500000033)=5.2398736793350074E-51; dx=-9.462400666675324E-57 evalInputDelta=8.224879553355929E-50
Fitness changed from 8.74886692128943E-50 to 5.2398736793350074E-51
Iteration 24 complete. Error: 5.2398736793350074E-51 Total: 0.5630; Orientation: 0.1152; Line Search: 0.3531
Zero gradient: 4.812191685720427E-29
th(0)=5.2398736793350074E-51;dx=-2.3157188820116806E-57
New Minimum: 5.2398736793350074E-51 > 2.0611500790058045E-51
WOLF (strong): th(7363790.444444923)=2.0611500790058045E-51; dx=1.4523799197447248E-57 evalInputDelta=3.178723600329203E-51
New Minimum: 2.0611500790058045E-51 > 1.820756354081335E-52
END: th(3681895.2222224614)=1.820756354081335E-52; dx=-4.31669481133478E-58 evalInputDelta=5.057798043926874E-51
Fitness changed from 5.2398736793350074E-51 to 1.820756354081335E-52
Iteration 25 complete. Error: 1.820756354081335E-52 Total: 0.7906; Orientation: 0.1129; Line Search: 0.5835
Zero gradient: 8.970330139059149E-30
th(0)=1.820756354081335E-52;dx=-8.046682280371292E-59
New Minimum: 1.820756354081335E-52 > 1.0319189752626257E-52
WOLF (strong): th(7932402.791818722)=1.0319189752626257E-52; dx=6.0577833312854084E-59 evalInputDelta=7.888373788187094E-53
New Minimum: 1.0319189752626257E-52 > 2.780888473427086E-54
END: th(3966201.395909361)=2.780888473427086E-54; dx=-9.944494745429417E-60 evalInputDelta=1.7929474693470643E-52
Fitness changed from 1.820756354081335E-52 to 2.780888473427086E-54
Iteration 26 complete. Error: 2.780888473427086E-54 Total: 0.7339; Orientation: 0.1131; Line Search: 0.5251
Zero gradient: 1.1085985232726834E-30
th(0)=2.780888473427086E-54;dx=-1.2289906858023743E-60
New Minimum: 2.780888473427086E-54 > 2.1937361427322353E-54
WOLF (strong): th(8544921.87500001)=2.1937361427322353E-54; dx=1.0915634888580253E-60 evalInputDelta=5.871523306948509E-55
New Minimum: 2.1937361427322353E-54 > 8.693041932380592E-57
END: th(4272460.937500005)=8.693041932380592E-57; dx=-6.871359847217448E-62 evalInputDelta=2.772195431494706E-54
Fitness changed from 2.780888473427086E-54 to 8.693041932380592E-57
Iteration 27 complete. Error: 8.693041932380592E-57 Total: 0.7384; Orientation: 0.1135; Line Search: 0.5286
Zero gradient: 6.198240122973076E-32
th(0)=8.693041932380592E-57;dx=-3.841818062203329E-63
Armijo: th(9204738.055556154)=9.293839850381197E-57; dx=3.972359075836618E-63 evalInputDelta=-6.007979180006042E-58
New Minimum: 8.693041932380592E-57 > 2.5091819592485244E-60
WOLF (strong): th(4602369.027778077)=2.5091819592485244E-60; dx=6.527050681664441E-65 evalInputDelta=8.690532750421343E-57
END: th(1534123.009259359)=3.7982128579242236E-57; dx=-2.539455205863338E-63 evalInputDelta=4.894829074456369E-57
Fitness changed from 8.693041932380592E-57 to 2.5091819592485244E-60
Iteration 28 complete. Error: 2.5091819592485244E-60 Total: 0.8836; Orientation: 0.1232; Line Search: 0.6634
Zero gradient: 1.0530490191034494E-33
th(0)=2.5091819592485244E-60;dx=-1.1089122366347368E-66
New Minimum: 2.5091819592485244E-60 > 1.8245110952249625E-61
END: th(3305167.829924468)=1.8245110952249625E-61; dx=-2.990228332035776E-67 evalInputDelta=2.3267308497260282E-60
Fitness changed from 2.5091819592485244E-60 to 1.8245110952249625E-61
Iteration 29 complete. Error: 1.8245110952249625E-61 Total: 0.5676; Orientation: 0.1125; Line Search: 0.3610
Zero gradient: 2.8395908241589865E-34
th(0)=1.8245110952249625E-61;dx=-8.063276048647913E-68
New Minimum: 1.8245110952249625E-61 > 6.000488568269458E-62
WOLF (strong): th(7120768.229166675)=6.000488568269458E-62; dx=4.624146451977098E-68 evalInputDelta=1.2244622383980166E-61
New Minimum: 6.000488568269458E-62 > 8.2977537870142E-63
END: th(3560384.1145833377)=8.2977537870142E-63; dx=-1.719564798335407E-68 evalInputDelta=1.7415335573548203E-61
Fitness changed from 1.8245110952249625E-61 to 8.2977537870142E-63
Iteration 30 complete. Error: 8.2977537870142E-63 Total: 0.8821; Orientation: 0.1237; Line Search: 0.6539
Zero gradient: 6.05567810582251E-35
th(0)=8.2977537870142E-63;dx=-3.66712373213381E-69
New Minimum: 8.2977537870142E-63 > 4.007820376681303E-63
WOLF (strong): th(7670615.046296796)=4.007820376681303E-63; dx=2.5485867218835086E-69 evalInputDelta=4.289933410332897E-63
New Minimum: 4.007820376681303E-63 > 1.9299681534655946E-64
END: th(3835307.523148398)=1.9299681534655946E-64; dx=-5.592685051251507E-70 evalInputDelta=8.10475697166764E-63
Fitness changed from 8.2977537870142E-63 to 1.9299681534655946E-64
Iteration 31 complete. Error: 1.9299681534655946E-64 Total: 0.7281; Orientation: 0.1131; Line Search: 0.5200
Zero gradient: 9.235439786461177E-36
th(0)=1.9299681534655946E-64;dx=-8.529334804935008E-71
New Minimum: 1.9299681534655946E-64 > 1.3163390351620021E-64
WOLF (strong): th(8262919.574811171)=1.3163390351620021E-64; dx=7.044074993925374E-71 evalInputDelta=6.1362911830359246E-65
New Minimum: 1.3163390351620021E-64 > 1.4630685894303883E-66
END: th(4131459.7874055854)=1.4630685894303883E-66; dx=-7.426299055048171E-72 evalInputDelta=1.9153374675712908E-64
Fitness changed from 1.9299681534655946E-64 to 1.4630685894303883E-66
Iteration 32 complete. Error: 1.4630685894303883E-66 Total: 0.7964; Orientation: 0.1118; Line Search: 0.5893
Zero gradient: 8.041088704768405E-37
th(0)=1.4630685894303883E-66;dx=-6.465910755795403E-73
New Minimum: 1.4630685894303883E-66 > 1.3676832317616862E-66
WOLF (strong): th(8900960.286458345)=1.3676832317616862E-66; dx=6.25158476264651E-73 evalInputDelta=9.538535766870201E-68
New Minimum: 1.3676832317616862E-66 > 4.018786852443026E-70
END: th(4450480.143229173)=4.018786852443026E-70; dx=-1.0716299657444663E-74 evalInputDelta=1.462666710745144E-66
Fitness changed from 1.4630685894303883E-66 to 4.018786852443026E-70
Iteration 33 complete. Error: 4.018786852443026E-70 Total: 0.7320; Orientation: 0.1120; Line Search: 0.5234
Final threshold in iteration 33: 4.018786852443026E-70 (> 0.0) after 30.118s (< 30.000s)

Returns

    4.018786852443026E-70

Training Converged

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:452 executed in 27.14 seconds (0.312 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 10805865560115
Reset training subject: 10805959572412
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=0.9435411534569775}, derivative=-4.169551618244071E-7}
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=0.9435411534569775}, derivative=-4.169551618244071E-7}, evalInputDelta = 0.0
New Minimum: 0.9435411534569775 > 0.9435411534569772
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=0.9435411534569772}, derivative=-4.1695516182440697E-7}, evalInputDelta = -3.3306690738754696E-16
New Minimum: 0.9435411534569772 > 0.9435411534569754
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=0.9435411534569754}, derivative=-4.1695516182440665E-7}, evalInputDelta = -2.1094237467877974E-15
New Minimum: 0.9435411534569754 > 0.9435411534569631
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=0.9435411534569631}, derivative=-4.169551618244039E-7}, evalInputDelta = -1.4432899320127035E-14
New Minimum: 0.9435411534569631 > 0.9435411534568774
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=0.9435411534568774}, derivative=-4.1695516182438495E-7}, evalInputDelta = -1.0014211682118912E-13
New Minimum: 0.9435411534568774 > 0.9435411534562768
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=0.9435411534562768}, derivative=-4.169551618242523E-7}, evalInputDelta = -7.007727731433988E-13
New Minimum: 0.9435411534562768 > 0.9435411534520721
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=0.9435411534520721}, derivative=-4.169551618233234E-7}, evalInputDelta = -4.905409412003792E-12
New Minimum: 0.9435411534520721 > 0.9435411534226394
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=0.9435411534226394}, derivative=-4.1695516181682115E-7}, evalInputDelta = -3.4338087928631467E-11
New Minimum: 0.9435411534226394 > 0.943541153216611
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=0.943541153216611}, derivative=-4.1695516177130526E-7}, evalInputDelta = -2.403665044781178E-10
New Minimum: 0.943541153216611 > 0.943541151774413
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=0.943541151774413}, derivative=-4.169551614526942E-7}, evalInputDelta = -1.6825645321461025E-9
New Minimum: 0.943541151774413 > 0.9435411416790263
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.9435411416790263}, derivative=-4.1695515922241683E-7}, evalInputDelta = -1.1777951280933507E-8
New Minimum: 0.9435411416790263 > 0.9435410710113201
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.9435410710113201}, derivative=-4.169551436104751E-7}, evalInputDelta = -8.244565741222232E-8
New Minimum: 0.9435410710113201 > 0.9435405763374514
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.9435405763374514}, derivative=-4.169550343268829E-7}, evalInputDelta = -5.771195261683459E-7
New Minimum: 0.9435405763374514 > 0.9435371136239997
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=0.9435371136239997}, derivative=-4.16954269341737E-7}, evalInputDelta = -4.039832977809077E-6
New Minimum: 0.9435371136239997 > 0.943512874807724
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=0.943512874807724}, derivative=-4.169489144456878E-7}, evalInputDelta = -2.827864925358803E-5
New Minimum: 0.943512874807724 > 0.9433432118101479
F(474.7561509943) = LineSearchPoint{point=PointSample{avg=0.9433432118101479}, derivative=-4.169114301719557E-7}, evalInputDelta = -1.9794164682962467E-4
New Minimum: 0.9433432118101479 > 0.9421559979285327
F(3323.2930569601003) = LineSearchPoint{point=PointSample{avg=0.9421559979285327}, derivative=-4.166490401878545E-7}, evalInputDelta = -0.0013851555284448702
New Minimum: 0.9421559979285327 > 0.9338664287523926
F(23263.0513987207) = LineSearchPoint{point=PointSample{avg=0.9338664287523926}, derivative=-4.1481230697746114E-7}, evalInputDelta = -0.00967472470458497
New Minimum: 0.9338664287523926 > 0.8768649249715754
F(162841.3597910449) = LineSearchPoint{point=PointSample{avg=0.8768649249715754}, derivative=-4.0195501486508804E-7}, evalInputDelta = -0.06667622848540211
New Minimum: 0.8768649249715754 > 0.52810561582696
F(1139889.5185373144) = LineSearchPoint{point=PointSample{avg=0.52810561582696}, derivative=-3.1194721902477636E-7}, evalInputDelta = -0.4154355376300175
F(7979226.6297612) = LineSearchPoint{point=PointSample{avg=0.5492848296241826}, derivative=3.1807062523945217E-7}, evalInputDelta = -0.39425632383279496
0.5492848296241826 <= 0.9435411534569775
New Minimum: 0.52810561582696 > 1.6956341955250528E-8
F(4526344.230636701) = LineSearchPoint{point=PointSample{avg=1.6956341955250528E-8}, derivative=5.427149697527908E-11}, evalInputDelta = -0.9435411365006355
Right bracket at 4526344.230636701
Converged to right
Fitness changed from 0.9435411534569775 to 1.6956341955250528E-8
Iteration 1 complete. Error: 1.6956341955250528E-8 Total: 5.1360; Orientation: 0.1137; Line Search: 4.7354
Low gradient: 8.656256169384601E-8
F(0.0) = LineSearchPoint{point=PointSample{avg=1.6956341955250528E-8}, derivative=-7.493077087000894E-15}
New Minimum: 1.6956341955250528E-8 > 2.377407435564906E-16
F(4526344.230636701) = LineSearchPoint{point=PointSample{avg=2.377407435564906E-16}, derivative=7.8719706404114525E-19}, evalInputDelta = -1.6956341717509784E-8
2.377407435564906E-16 <= 1.6956341955250528E-8
Converged to right
Fitness changed from 1.6956341955250528E-8 to 2.377407435564906E-16
Iteration 2 complete. Error: 2.377407435564906E-16 Total: 0.5544; Orientation: 0.1127; Line Search: 0.3450
Zero gradient: 1.0250037699347381E-11
F(0.0) = LineSearchPoint{point=PointSample{avg=2.377407435564906E-16}, derivative=-1.0506327283804255E-22}
New Minimum: 2.377407435564906E-16 > 5.645283745982354E-24
F(4526344.230636701) = LineSearchPoint{point=PointSample{avg=5.645283745982354E-24}, derivative=1.5699874724642998E-26}, evalInputDelta = -2.3774073791120687E-16
5.645283745982354E-24 <= 2.377407435564906E-16
Converged to right
Fitness changed from 2.377407435564906E-16 to 5.645283745982354E-24
Iteration 3 complete. Error: 5.645283745982354E-24 Total: 0.6172; Orientation: 0.1642; Line Search: 0.3584
Zero gradient: 1.579501106998623E-15
F(0.0) = LineSearchPoint{point=PointSample{avg=5.645283745982354E-24}, derivative=-2.4948237470098757E-30}
New Minimum: 5.645283745982354E-24 > 1.6045611597361878E-31
F(4526344.230636701) = LineSearchPoint{point=PointSample{avg=1.6045611597361878E-31}, derivative=4.091257274108754E-34}, evalInputDelta = -5.6452835855262375E-24
1.6045611597361878E-31 <= 5.645283745982354E-24
Converged to right
Fitness changed from 5.645283745982354E-24 to 1.6045611597361878E-31
Iteration 4 complete. Error: 1.6045611597361878E-31 Total: 0.5677; Orientation: 0.1140; Line Search: 0.3583
Zero gradient: 2.6629095519449993E-19
F(0.0) = LineSearchPoint{point=PointSample{avg=1.6045611597361878E-31}, derivative=-7.091087281839913E-38}
New Minimum: 1.6045611597361878E-31 > 1.0709202697552869E-33
F(4526344.230636701) = LineSearchPoint{point=PointSample{avg=1.0709202697552869E-33}, derivative=5.1494839109186946E-40}, evalInputDelta = -1.5938519570386349E-31
1.0709202697552869E-33 <= 1.6045611597361878E-31
Converged to right
Fitness changed from 1.6045611597361878E-31 to 1.0709202697552869E-33
Iteration 5 complete. Error: 1.0709202697552869E-33 Total: 0.6138; Orientation: 0.1472; Line Search: 0.3592
Zero gradient: 2.17540748462974E-20
F(0.0) = LineSearchPoint{point=PointSample{avg=1.0709202697552869E-33}, derivative=-4.732397724183096E-40}
New Minimum: 1.0709202697552869E-33 > 4.363185261212673E-34
F(4526344.230636701) = LineSearchPoint{point=PointSample{avg=4.363185261212673E-34}, derivative=1.928008147058284E-40}, evalInputDelta = -6.346017436340196E-34
4.363185261212673E-34 <= 1.0709202697552869E-33
New Minimum: 4.363185261212673E-34 > 1.08641140227925E-42
F(3216089.4621189237) = LineSearchPoint{point=PointSample{avg=1.08641140227925E-42}, derivative=-1.6594110774245265E-48}, evalInputDelta = -1.0709202686688755E-33
Left bracket at 3216089.4621189237
Converged to left
Fitness changed from 1.0709202697552869E-33 to 1.08641140227925E-42
Iteration 6 complete. 

...skipping 9555 bytes...

351763854E-202 Total: 2.0133; Orientation: 0.1155; Line Search: 1.8029
Zero gradient: 1.490557884678629E-104
F(0.0) = LineSearchPoint{point=PointSample{avg=5.027275351763854E-202}, derivative=-2.2217628075776297E-208}
New Minimum: 5.027275351763854E-202 > 6.196595288730804E-234
F(4525483.399593904) = LineSearchPoint{point=PointSample{avg=6.196595288730804E-234}, derivative=-2.4666522242285158E-224}, evalInputDelta = -5.027275351763854E-202
F(3.1678383797157325E7) = LineSearchPoint{point=PointSample{avg=1.8098191266349867E-200}, derivative=1.3330576845465775E-207}, evalInputDelta = 1.7595463731173482E-200
F(2436798.7536274865) = LineSearchPoint{point=PointSample{avg=1.0708988914999926E-202}, derivative=-1.0254289881127522E-208}, evalInputDelta = -3.956376460263861E-202
F(1.7057591275392406E7) = LineSearchPoint{point=PointSample{avg=3.8552360093999724E-201}, derivative=6.152573928676513E-208}, evalInputDelta = 3.352508474223587E-201
F(1312122.405799416) = LineSearchPoint{point=PointSample{avg=2.534671932544362E-202}, derivative=-1.5775830586350037E-208}, evalInputDelta = -2.492603419219492E-202
F(9184856.840595912) = LineSearchPoint{point=PointSample{avg=5.3291477381745185E-202}, derivative=2.2874954350207548E-208}, evalInputDelta = 3.0187238641066464E-203
F(706527.4492766086) = LineSearchPoint{point=PointSample{avg=3.580074124131186E-202}, derivative=-1.874896788916216E-208}, evalInputDelta = -1.447201227632668E-202
F(4945692.144936261) = LineSearchPoint{point=PointSample{avg=4.334438985238586E-204}, derivative=2.0629932305226978E-209}, evalInputDelta = -4.983930961911468E-202
4.334438985238586E-204 <= 5.027275351763854E-202
F(4525483.399593904) = LineSearchPoint{point=PointSample{avg=6.196595288730804E-234}, derivative=-2.4666522242285158E-224}, evalInputDelta = -5.027275351763854E-202
Left bracket at 4525483.399593904
Converged to left
Fitness changed from 5.027275351763854E-202 to 6.196595288730804E-234
Iteration 12 complete. Error: 6.196595288730804E-234 Total: 1.9909; Orientation: 0.1141; Line Search: 1.7823
Zero gradient: 1.654851683106783E-120
F(0.0) = LineSearchPoint{point=PointSample{avg=6.196595288730804E-234}, derivative=-2.7385340930813526E-240}
New Minimum: 6.196595288730804E-234 > 7.637893388681936E-266
F(4525483.399593904) = LineSearchPoint{point=PointSample{avg=7.637893388681936E-266}, derivative=-3.040383603859889E-256}, evalInputDelta = -6.196595288730804E-234
F(3.1678383797157325E7) = LineSearchPoint{point=PointSample{avg=2.2307743039430884E-232}, derivative=1.6431204558488111E-239}, evalInputDelta = 2.1688083510557804E-232
F(2436798.7536274865) = LineSearchPoint{point=PointSample{avg=1.3199847952325973E-234}, derivative=-1.2639388121913935E-240}, evalInputDelta = -4.876610493498206E-234
F(1.7057591275392406E7) = LineSearchPoint{point=PointSample{avg=4.751945262837349E-233}, derivative=7.58363287314836E-240}, evalInputDelta = 4.132285733964269E-233
F(1312122.405799416) = LineSearchPoint{point=PointSample{avg=3.1242243674144324E-234}, derivative=-1.9445212495252212E-240}, evalInputDelta = -3.072370921316371E-234
F(9184856.840595912) = LineSearchPoint{point=PointSample{avg=6.568681732488841E-234}, derivative=2.81955581181157E-240}, evalInputDelta = 3.7208644375803745E-235
F(706527.4492766086) = LineSearchPoint{point=PointSample{avg=4.4127820536257003E-234}, derivative=-2.3109887157818975E-240}, evalInputDelta = -1.7838132351051033E-234
F(4945692.144936261) = LineSearchPoint{point=PointSample{avg=5.342608533625867E-236}, derivative=2.542835480148367E-241}, evalInputDelta = -6.143169203394545E-234
5.342608533625867E-236 <= 6.196595288730804E-234
F(4525483.399593904) = LineSearchPoint{point=PointSample{avg=7.637893388681936E-266}, derivative=-3.040383603859889E-256}, evalInputDelta = -6.196595288730804E-234
Left bracket at 4525483.399593904
Converged to left
Fitness changed from 6.196595288730804E-234 to 7.637893388681936E-266
Iteration 13 complete. Error: 7.637893388681936E-266 Total: 2.1710; Orientation: 0.1126; Line Search: 1.9647
Zero gradient: 1.8372544409248436E-136
F(0.0) = LineSearchPoint{point=PointSample{avg=7.637893388681936E-266}, derivative=-3.37550388069806E-272}
New Minimum: 7.637893388681936E-266 > 9.414430457151895E-298
F(4525483.399593904) = LineSearchPoint{point=PointSample{avg=9.414430457151895E-298}, derivative=-3.747562128062554E-288}, evalInputDelta = -7.637893388681936E-266
F(3.1678383797157325E7) = LineSearchPoint{point=PointSample{avg=2.7496416199254957E-264}, derivative=2.0253023284188355E-271}, evalInputDelta = 2.6732626860386764E-264
F(2436798.7536274865) = LineSearchPoint{point=PointSample{avg=1.6270068756955603E-266}, derivative=-1.5579248680144892E-272}, evalInputDelta = -6.010886512986375E-266
F(1.7057591275392406E7) = LineSearchPoint{point=PointSample{avg=5.857224752504016E-265}, derivative=9.347549208086935E-272}, evalInputDelta = 5.093435413635823E-265
F(1312122.405799416) = LineSearchPoint{point=PointSample{avg=3.850903847800144E-266}, derivative=-2.396807489253061E-272}, evalInputDelta = -3.786989540881792E-266
F(9184856.840595912) = LineSearchPoint{point=PointSample{avg=8.096525339999799E-266}, derivative=3.4753708594169374E-272}, evalInputDelta = 4.586319513178633E-267
F(706527.4492766086) = LineSearchPoint{point=PointSample{avg=5.439173820884696E-266}, derivative=-2.84851351607383E-272}, evalInputDelta = -2.1987195677972396E-266
F(4945692.144936261) = LineSearchPoint{point=PointSample{avg=6.585273443871256E-268}, derivative=3.134286716715542E-273}, evalInputDelta = -7.572040654243224E-266
6.585273443871256E-268 <= 7.637893388681936E-266
F(4525483.399593904) = LineSearchPoint{point=PointSample{avg=9.414430457151895E-298}, derivative=-3.747562128062554E-288}, evalInputDelta = -7.637893388681936E-266
Left bracket at 4525483.399593904
Converged to left
Fitness changed from 7.637893388681936E-266 to 9.414430457151895E-298
Iteration 14 complete. Error: 9.414430457151895E-298 Total: 2.3030; Orientation: 0.1133; Line Search: 2.0950
Zero gradient: 2.039762182409581E-152
F(0.0) = LineSearchPoint{point=PointSample{avg=9.414430457151895E-298}, derivative=-4.160629760788297E-304}
New Minimum: 9.414430457151895E-298 > 0.0
F(4525483.399593904) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-4.678E-320}, evalInputDelta = -9.414430457151895E-298
F(3.1678383797157325E7) = LineSearchPoint{point=PointSample{avg=3.389194964574681E-296}, derivative=2.4963778564729774E-303}, evalInputDelta = 3.295050660003162E-296
F(2436798.7536274865) = LineSearchPoint{point=PointSample{avg=2.0054408074406406E-298}, derivative=-1.9202906588253677E-304}, evalInputDelta = -7.4089896497112544E-298
F(1.7057591275392406E7) = LineSearchPoint{point=PointSample{avg=7.219586906786304E-297}, derivative=1.1521743952952205E-303}, evalInputDelta = 6.278143861071115E-297
F(1312122.405799416) = LineSearchPoint{point=PointSample{avg=4.746605461397967E-298}, derivative=-2.954293321269797E-304}, evalInputDelta = -4.667824995753928E-298
F(9184856.840595912) = LineSearchPoint{point=PointSample{avg=9.979737982589221E-298}, derivative=4.283725315841205E-304}, evalInputDelta = 5.65307525437326E-299
F(706527.4492766086) = LineSearchPoint{point=PointSample{avg=6.704299349996142E-298}, derivative=-3.511063985662951E-304}, evalInputDelta = -2.7101311071557535E-298
F(4945692.144936261) = LineSearchPoint{point=PointSample{avg=8.116976203219014E-300}, derivative=3.8633066508912743E-305}, evalInputDelta = -9.333260695119706E-298
8.116976203219014E-300 <= 9.414430457151895E-298
F(4525483.399593895) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=-7.85204E-319}, evalInputDelta = -9.414430457151895E-298
Left bracket at 4525483.399593895
Converged to left
Fitness changed from 9.414430457151895E-298 to 0.0
Iteration 15 complete. Error: 0.0 Total: 1.9959; Orientation: 0.1117; Line Search: 1.7905
Zero gradient: 0.0
F(0.0) = LineSearchPoint{point=PointSample{avg=0.0}, derivative=0.0}
Fitness changed from 0.0 to 0.0
Static Iteration Total: 0.3759; Orientation: 0.1143; Line Search: 0.1670
Iteration 16 failed. Error: 0.0
Previous Error: 0.0 -> 0.0
Optimization terminated 16
Final threshold in iteration 16: 0.0 (> 0.0) after 27.143s (< 30.000s)

Returns

    0.0

Training Converged

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:509 executed in 45.79 seconds (0.666 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 10833041009062
Reset training subject: 10833135067362
Adding measurement 3fa11ac to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 0.9435411534569775 < 0.9435411534569775. Total: 1
th(0)=0.9435411534569775;dx=-4.169551618244071E-7
Adding measurement 4edbc799 to history. Total: 1
New Minimum: 0.9435411534569775 > 0.9435402551545264
WOLFE (weak): th(2.154434690031884)=0.9435402551545264; dx=-4.1695496337097924E-7 evalInputDelta=8.983024510911264E-7
Adding measurement 43028625 to history. Total: 2
New Minimum: 0.9435402551545264 > 0.9435393568525029
WOLFE (weak): th(4.308869380063768)=0.9435393568525029; dx=-4.169547649175514E-7 evalInputDelta=1.796604474635366E-6
Adding measurement 446e5e7c to history. Total: 3
New Minimum: 0.9435393568525029 > 0.9435357636486845
WOLFE (weak): th(12.926608140191302)=0.9435357636486845; dx=-4.1695397110383903E-7 evalInputDelta=5.38980829301039E-6
Adding measurement 5d638913 to history. Total: 4
New Minimum: 0.9435357636486845 > 0.9435195943161576
WOLFE (weak): th(51.70643256076521)=0.9435195943161576; dx=-4.169503989421203E-7 evalInputDelta=2.1559140819915612E-5
Adding measurement 681dd00a to history. Total: 5
New Minimum: 0.9435195943161576 > 0.9434333602155945
WOLFE (weak): th(258.53216280382605)=0.9434333602155945; dx=-4.169313474125812E-7 evalInputDelta=1.0779324138299184E-4
Adding measurement 6eb03d9e to history. Total: 6
New Minimum: 0.9434333602155945 > 0.9428944863605574
WOLFE (weak): th(1551.1929768229563)=0.9428944863605574; dx=-4.168122753387498E-7 evalInputDelta=6.466670964201393E-4
Adding measurement 66608721 to history. Total: 7
New Minimum: 0.9428944863605574 > 0.9390191383192983
WOLFE (weak): th(10858.350837760694)=0.9390191383192983; dx=-4.159549556847592E-7 evalInputDelta=0.004522015137679247
Adding measurement 11e8f5fa to history. Total: 8
New Minimum: 0.9390191383192983 > 0.9076691301549058
WOLFE (weak): th(86866.80670208555)=0.9076691301549058; dx=-4.0895346484413216E-7 evalInputDelta=0.03587202330207173
Adding measurement 600e3ab2 to history. Total: 9
New Minimum: 0.9076691301549058 > 0.6457167927012458
END: th(781801.26031877)=0.6457167927012458; dx=-3.449363124072154E-7 evalInputDelta=0.2978243607557317
Fitness changed from 0.9435411534569775 to 0.6457167927012458
Iteration 1 complete. Error: 0.6457167927012458 Total: 2.9287; Orientation: 0.1475; Line Search: 2.4978
Non-optimal measurement 0.6457167927012458 < 0.6457167927012458. Total: 10
Rejected: LBFGS Orientation magnitude: 2.418e+03, gradient 5.342e-04, dot -1.000; [6b406d61-0032-46c4-9eb5-c1fa641c403d = 1.000/1.000e+00, 6ea6f2a8-2c66-4169-95df-9df1acaa5a25 = 1.000/1.000e+00, 64dcd0ef-3f09-4a7d-a20c-a18c9bd7ff46 = 1.000/1.000e+00, 9dbf79d3-f484-436a-8f3e-6cbee38837b7 = 1.000/1.000e+00, c8c85710-1e69-46df-a759-9ec65b3fbf4a = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.6457167927012458, 0.9076691301549058, 0.9390191383192983, 0.9428944863605574, 0.9434333602155945, 0.9435195943161576, 0.9435357636486845, 0.9435393568525029, 0.9435402551545264, 0.9435411534569775
Rejected: LBFGS Orientation magnitude: 2.418e+03, gradient 5.342e-04, dot -1.000; [64dcd0ef-3f09-4a7d-a20c-a18c9bd7ff46 = 1.000/1.000e+00, 6b406d61-0032-46c4-9eb5-c1fa641c403d = 1.000/1.000e+00, 6ea6f2a8-2c66-4169-95df-9df1acaa5a25 = 1.000/1.000e+00, c8c85710-1e69-46df-a759-9ec65b3fbf4a = 1.000/1.000e+00, 9dbf79d3-f484-436a-8f3e-6cbee38837b7 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.6457167927012458, 0.9076691301549058, 0.9390191383192983, 0.9428944863605574, 0.9434333602155945, 0.9435195943161576, 0.9435357636486845, 0.9435393568525029, 0.9435402551545264
Rejected: LBFGS Orientation magnitude: 2.418e+03, gradient 5.342e-04, dot -1.000; [64dcd0ef-3f09-4a7d-a20c-a18c9bd7ff46 = 1.000/1.000e+00, 6b406d61-0032-46c4-9eb5-c1fa641c403d = 1.000/1.000e+00, c8c85710-1e69-46df-a759-9ec65b3fbf4a = 1.000/1.000e+00, 6ea6f2a8-2c66-4169-95df-9df1acaa5a25 = 1.000/1.000e+00, 9dbf79d3-f484-436a-8f3e-6cbee38837b7 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.6457167927012458, 0.9076691301549058, 0.9390191383192983, 0.9428944863605574, 0.9434333602155945, 0.9435195943161576, 0.9435357636486845, 0.9435393568525029
Rejected: LBFGS Orientation magnitude: 2.418e+03, gradient 5.342e-04, dot -1.000; [9dbf79d3-f484-436a-8f3e-6cbee38837b7 = 1.000/1.000e+00, 6ea6f2a8-2c66-4169-95df-9df1acaa5a25 = 1.000/1.000e+00, 64dcd0ef-3f09-4a7d-a20c-a18c9bd7ff46 = 1.000/1.000e+00, c8c85710-1e69-46df-a759-9ec65b3fbf4a = 1.000/1.000e+00, 6b406d61-0032-46c4-9eb5-c1fa641c403d = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.6457167927012458, 0.9076691301549058, 0.9390191383192983, 0.9428944863605574, 0.9434333602155945, 0.9435195943161576, 0.9435357636486845
Rejected: LBFGS Orientation magnitude: 2.418e+03, gradient 5.342e-04, dot -1.000; [c8c85710-1e69-46df-a759-9ec65b3fbf4a = 1.000/1.000e+00, 6ea6f2a8-2c66-4169-95df-9df1acaa5a25 = 1.000/1.000e+00, 9dbf79d3-f484-436a-8f3e-6cbee38837b7 = 1.000/1.000e+00, 64dcd0ef-3f09-4a7d-a20c-a18c9bd7ff46 = 1.000/1.000e+00, 6b406d61-0032-46c4-9eb5-c1fa641c403d = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.6457167927012458, 0.9076691301549058, 0.9390191383192983, 0.9428944863605574, 0.9434333602155945, 0.9435195943161576
Rejected: LBFGS Orientation magnitude: 2.418e+03, gradient 5.342e-04, dot -1.000; [6b406d61-0032-46c4-9eb5-c1fa641c403d = 1.000/1.000e+00, 9dbf79d3-f484-436a-8f3e-6cbee38837b7 = 1.000/1.000e+00, c8c85710-1e69-46df-a759-9ec65b3fbf4a = 1.000/1.000e+00, 6ea6f2a8-2c66-4169-95df-9df1acaa5a25 = 1.000/1.000e+00, 64dcd0ef-3f09-4a7d-a20c-a18c9bd7ff46 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.6457167927012458, 0.9076691301549058, 0.9390191383192983, 0.9428944863605574, 0.9434333602155945
Rejected: LBFGS Orientation magnitude: 2.418e+03, gradient 5.342e-04, dot -1.000; [64dcd0ef-3f09-4a7d-a20c-a18c9bd7ff46 = 1.000/1.000e+00, 6ea6f2a8-2c66-4169-95df-9df1acaa5a25 = 1.000/1.000e+00, 9dbf79d3-f484-436a-8f3e-6cbee38837b7 = 1.000/1.000e+00, 6b406d61-0032-46c4-9eb5-c1fa641c403d = 1.000/1.000e+00, c8c85710-1e69-46df-a759-9ec65b3fbf4a = 1.000/1.000e+00]
Orientation rejected. Popping history element from 0.6457167927012458, 0.9076691301549058, 0.9390191383192983, 0.9428944863605574
LBFGS Accumulation History: 3 points
Removed measurement 600e3ab2 to history. Total: 9
Removed measurement 11e8f5fa to history. Total: 8
Removed measurement 66608721 to history. Total: 7
Removed measurement 6eb03d9e to history. Total: 6
Removed measurement 681dd00a to history. Total: 5
Removed measurement 5d638913 to history. Total: 4
Removed measurement 446e5e7c to history. Total: 3
Adding measurement 412e937 to history. Total: 3
th(0)=0.6457167927012458;dx=-2.8535696539382136E-7
Adding measurement 6081abe5 to history. Total: 4
New Minimum: 0.6457167927012458 > 0.2545162161702924
END: th(1684339.7559414052)=0.2545162161702924; dx=-1.7915620239784022E-7 evalInputDelta=0.3912005765309534
Fitness changed from 0.6457167927012458 to 0.2545162161702924
Iteration 2 complete. Error: 0.2545162161702924 Total: 42.8586; Orientation: 42.2842; Line Search: 0.4789
Final threshold in iteration 2: 0.2545162161702924 (> 0.0) after 45.788s (< 30.000s)

Returns

    0.2545162161702924

This training apply resulted in the following configuration:

TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> state = network.state();
    assert state != null;
    String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
        .orElse("");
    state.freeRef();
    return description;

Returns

    

And regressed input:

TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
      return RefArrays.stream(x);
    }).limit(1).map(x -> {
      String temp_18_0015 = x.prettyPrint();
      x.freeRef();
      return temp_18_0015;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [
    	[ [ -0.2039235088355057 ], [ 1.2087016966804973 ], [ -0.5326730016690944 ], [ 0.5968216845066792 ], [ 0.6762349770216904 ], [ 0.8353480889372474 ], [ 0.21255466167594517 ], [ -0.8816008081325044 ], ... ],
    	[ [ -0.5055310496846679 ], [ -1.3514088034333185 ], [ -1.1088494133958278 ], [ 0.1726011650485243 ], [ -0.12267658908339085 ], [ -0.7576972068615941 ], [ -0.02525424484735006 ], [ -1.0916275234197492 ], ... ],
    	[ [ -1.397165097582398 ], [ -0.31405305439798825 ], [ 0.03309824183291976 ], [ 0.21197605190289848 ], [ -0.42281746032096035 ], [ 1.1840054245573022 ], [ 1.4562839122830527 ], [ 0.10265656254697597 ], ... ],
    	[ [ 0.13771508409618982 ], [ -0.8044090678853018 ], [ 0.5222736052059197 ], [ 0.7055785523202975 ], [ -0.06754535398623918 ], [ -0.8183215810358316 ], [ -1.2903560876830213 ], [ -0.4834572671261935 ], ... ],
    	[ [ -0.15525842103145215 ], [ 0.7344877653750622 ], [ -0.7873699661104434 ], [ 0.22766995652657518 ], [ -0.12980660104866665 ], [ 0.6549763462186565 ], [ 0.3933677009338786 ], [ -0.9482451653359012 ], ... ],
    	[ [ 1.1485303430412728 ], [ -0.9386759394836144 ], [ -1.6676537756075982 ], [ 0.0070773995281978586 ], [ 0.16171560448410305 ], [ -1.169865704177773 ], [ -0.02633515134694031 ], [ 1.661205966847073 ], ... ],
    	[ [ 0.03743001755351283 ], [ -0.10855530425536086 ], [ 1.2666361477819228 ], [ 0.854539212180331 ], [ -0.013429787146084282 ], [ 1.3420775449941975 ], [ -0.015237739904444741 ], [ -0.4527511963397174 ], ... ],
    	[ [ -0.6935128953457763 ], [ 0.06036901896291447 ], [ -0.579609558539913 ], [ -0.6021905689387621 ], [ -0.5158268101337535 ], [ 0.4590046606528181 ], [ 0.8957333614141332 ], [ -0.9592874636294013 ], ... ],
    	...
    ]

To produce the following output:

TrainingTester.java:633 executed in 0.02 seconds (0.000 gc):

    Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
    @Nullable
    Result eval = layer.eval(array);
    assert eval != null;
    TensorList tensorList = Result.getData(eval);
    String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
      String temp_18_0017 = x.prettyPrint();
      x.freeRef();
      return temp_18_0017;
    }).reduce((a, b) -> a + "\n" + b).orElse("");
    tensorList.freeRef();
    return temp_18_0016;

Returns

    [
    	[ [ -0.12125360599445394 ], [ 0.7186904539846162 ], [ -0.31672858776208834 ], [ 0.3548713487735584 ], [ 0.40209034399424465 ], [ 0.49669834594157214 ], [ 0.12638571517070685 ], [ -0.524199921174719 ], ... ],
    	[ [ -0.30058998442555923 ], [ -0.8035414758420771 ], [ -0.6593197259632604 ], [ 0.10262924383589953 ], [ -0.0729439280578197 ], [ -0.45052751483420495 ], [ -0.015016263756417056 ], [ -0.6490798078276125 ], ... ],
    	[ [ -0.8307471748921933 ], [ -0.18673692525620808 ], [ 0.019680332179196857 ], [ 0.1260416720900302 ], [ -0.25140842898978977 ], [ 0.7040064356204834 ], [ 0.8658978222826217 ], [ 0.06103995246683095 ], ... ],
    	[ [ 0.08188586727767982 ], [ -0.47830217222626764 ], [ 0.3105451083399537 ], [ 0.419537950833042 ], [ -0.04016270639862662 ], [ -0.48657447950582117 ], [ -0.7672407391727011 ], [ -0.28746490701246885 ], ... ],
    	[ [ -0.09231719278742875 ], [ 0.4367272712175108 ], [ -0.46817080609685907 ], [ 0.13537331346022952 ], [ -0.07718345701678044 ], [ 0.38945001250359673 ], [ 0.2338975629348999 ], [ -0.5638261463844747 ], ... ],
    	[ [ 0.6829134715356181 ], [ -0.5581363646049134 ], [ -0.991572185497896 ], [ 0.004208246935743978 ], [ 0.0961566548696106 ], [ -0.6955991696117356 ], [ -0.015658974596776492 ], [ 0.9877385344368299 ], ... ],
    	[ [ 0.022256021360800918 ], [ -0.06454736439106117 ], [ 0.753137297233689 ], [ 0.5081092727869317 ], [ -0.007985399202745958 ], [ 0.7979933027957451 ], [ -0.00906041433968534 ], [ -0.26920705813386714 ], ... ],
    	[ [ -0.41236374726763186 ], [ 0.03589563245707575 ], [ -0.34463703712102844 ], [ -0.3580636807411046 ], [ -0.3067118442923712 ], [ 0.27292537287379753 ], [ 0.532603038314268 ], [ -0.5703918018472396 ], ... ],
    	...
    ]

TrainingTester.java:432 executed in 0.15 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging
Plotting range=[1.0, -297.0262059482759], [33.0, -0.18995791905698403]; valueStats=DoubleSummaryStatistics{count=49, sum=1.810646, min=0.000000, average=0.036952, max=0.645717}
Plotting 33 points for GD
Plotting 15 points for CjGD
Plotting 2 points for LBFGS

Returns

Result

TrainingTester.java:435 executed in 0.02 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging
Plotting range=[0.0, -297.0262059482759], [42.859, -0.18995791905698403]; valueStats=DoubleSummaryStatistics{count=49, sum=1.810646, min=0.000000, average=0.036952, max=0.645717}
Plotting 33 points for GD
Plotting 15 points for CjGD
Plotting 2 points for LBFGS

Returns

Result

Results

TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):

    return grid(inputLearning, modelLearning, completeLearning);

Returns

Result

TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):

    return new ComponentResult(null == inputLearning ? null : inputLearning.value,
        null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);

Returns

    {"input":{ "LBFGS": { "type": "NonConverged", "value": 0.2545162161702924 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 4.018786852443026E-70 } }, "model":null, "complete":null}

LayerTests.java:425 executed in 0.00 seconds (0.000 gc):

    throwException(exceptions.addRef());

Results

detailsresult
{"input":{ "LBFGS": { "type": "NonConverged", "value": 0.2545162161702924 }, "CjGD": { "type": "Converged", "value": 0.0 }, "GD": { "type": "Converged", "value": 4.018786852443026E-70 } }, "model":null, "complete":null}OK
  {
    "result": "OK",
    "performance": {
      "execution_time": "104.383",
      "gc_time": "2.069"
    },
    "created_on": 1586745407042,
    "file_name": "trainingTest",
    "report": {
      "simpleName": "LRNLayerTest",
      "canonicalName": "com.simiacryptus.mindseye.layers.cudnn.LRNLayerTest",
      "link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/test/java/com/simiacryptus/mindseye/layers/cudnn/LRNLayerTest.java",
      "javaDoc": ""
    },
    "training_analysis": {
      "input": {
        "LBFGS": {
          "type": "NonConverged",
          "value": 0.2545162161702924
        },
        "CjGD": {
          "type": "Converged",
          "value": 0.0
        },
        "GD": {
          "type": "Converged",
          "value": 4.018786852443026E-70
        }
      }
    },
    "archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/cudnn/LRNLayer/LRNLayerTest/trainingTest/202004133647",
    "id": "6bbc92f8-f4c5-4a2c-9286-c488df0fbe96",
    "report_type": "Components",
    "display_name": "Comparative Training",
    "target": {
      "simpleName": "LRNLayer",
      "canonicalName": "com.simiacryptus.mindseye.layers.cudnn.LRNLayer",
      "link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/main/java/com/simiacryptus/mindseye/layers/cudnn/LRNLayer.java",
      "javaDoc": ""
    }
  }