1. Test Modules
  2. Network Diagram
  3. Training Characteristics
    1. Input Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    2. Model Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    3. Composite Learning
      1. Gradient Descent
      2. Conjugate Gradient Descent
      3. Limited-Memory BFGS
    4. Results
  4. Results

Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase

Test Modules

Network Diagram

This is a network apply the following layout:

LayerTests.java:203 executed in 10.01 seconds (0.000 gc):

    return Graphviz.fromGraph((Graph) TestUtil.toGraph(((DAGNetwork) layer).addRef())).height(400).width(600)
        .render(Format.PNG).toImage();
java.lang.reflect.InvocationTargetException
java.lang.RuntimeException: java.lang.reflect.InvocationTargetException
at com.simiacryptus.util.Util.throwException(Util.java:502)
at com.simiacryptus.notebook.MarkdownNotebookOutput.lambda$eval$dc58be99$1(MarkdownNotebookOutput.java:659)
at com.simiacryptus.util.test.SysOutInterceptor.withOutput(SysOutInterceptor.java:102)
at com.simiacryptus.notebook.MarkdownNotebookOutput.eval(MarkdownNotebookOutput.java:649)
at com.simiacryptus.notebook.NotebookOutput.eval(NotebookOutput.java:125)
at com.simiacryptus.mindseye.test.unit.LayerTests.renderGraph(LayerTests.java:203)
at com.simiacryptus.mindseye.test.unit.LayerTests.run(LayerTests.java:415)
at com.simiacryptus.mindseye.test.LayerTestBase.trainingTest(LayerTestBase.java:95)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:686)
at org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
at org.junit.jupiter.engine.extension.TimeoutInvocation.proceed(TimeoutInvocation.java:46)
at org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestableMethod(TimeoutExtension.java:140)
at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestMethod(TimeoutExtension.java:84)
at org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
at org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
at org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
at org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
at org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeTestMethod$6(TestMethodTestDescriptor.java:205)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeTestMethod(TestMethodTestDescriptor.java:201)
at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:137)
at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:71)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:135)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:125)
at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:135)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:123)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:122)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:80)
at java.util.ArrayList.forEach(Unknown Source)
at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:125)
at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:135)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:123)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:122)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:80)
at java.util.ArrayList.forEach(Unknown Source)
at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:125)
at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:135)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:123)
at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:122)
at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:80)
at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
at org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
at org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:248)
at org.junit.platform.launcher.core.DefaultLauncher.lambda$execute$5(DefaultLauncher.java:211)
at org.junit.platform.launcher.core.DefaultLauncher.withInterceptedStreams(DefaultLauncher.java:226)
at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:199)
at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:141)
at org.junit.platform.runner.JUnitPlatform.run(JUnitPlatform.java:139)
at com.simiacryptus.util.test.MacroTestRunner.runTest(MacroTestRunner.java:94)
at com.simiacryptus.util.test.MacroTestRunner.lambda$null$8bb7732f$1(MacroTestRunner.java:177)
at com.simiacryptus.aws.TendrilControl.lambda$null$0(TendrilControl.java:86)
at java.lang.Thread.run(Unknown Source)
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at com.simiacryptus.ref.lang.RefUtil$RefWrapperHandler.invoke(RefUtil.java:238)
at com.sun.proxy.$Proxy44.get(Unknown Source)
at com.simiacryptus.notebook.MarkdownNotebookOutput.lambda$eval$dc58be99$1(MarkdownNotebookOutput.java:657)
... 71 more
Caused by: guru.nidi.graphviz.engine.GraphvizException: Engine took too long to respond, try setting a higher timout
at guru.nidi.graphviz.engine.GraphvizServerEngine.execute(GraphvizServerEngine.java:49)
at guru.nidi.graphviz.engine.Graphviz.execute(Graphviz.java:246)
at guru.nidi.graphviz.engine.Renderer.execute(Renderer.java:134)
at guru.nidi.graphviz.engine.Renderer.toImage(Renderer.java:105)
at com.simiacryptus.mindseye.test.unit.LayerTests.lambda$renderGraph$12c65ef7$1(LayerTests.java:205)
... 77 more


Using Seed 1344533435234331648

Training Characteristics

Input Learning

In this apply, we use a network to learn this target input, given it's pre-evaluated output:

TrainingTester.java:332 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
      try {
        return x.prettyPrint();
      } finally {
        x.freeRef();
      }
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [ 0.7, -0.128, 0.08 ]
    [ 0.7, 0.08, -0.128 ]
    [ -0.128, 0.08, 0.7 ]
    [ 0.7, 0.08, -0.128 ]
    [ 0.08, 0.7, -0.128 ]

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:480 executed in 0.23 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 3724249220304
Reset training subject: 3724271347165
Constructing line search parameters: GD
th(0)=0.28015023395400235;dx=-0.0385718211016262
New Minimum: 0.28015023395400235 > 0.1942286992847361
WOLFE (weak): th(2.154434690031884)=0.1942286992847361; dx=-0.04064599841669012 evalInputDelta=0.08592153466926625
New Minimum: 0.1942286992847361 > 0.10877425370162033
WOLFE (weak): th(4.308869380063768)=0.10877425370162033; dx=-0.0374337486798352 evalInputDelta=0.17137598025238202
New Minimum: 0.10877425370162033 > 1.3392379381061466E-9
END: th(12.926608140191302)=1.3392379381061466E-9; dx=-5.540135503207175E-7 evalInputDelta=0.2801502326147644
Fitness changed from 0.28015023395400235 to 1.3392379381061466E-9
Iteration 1 complete. Error: 1.3392379381061466E-9 Total: 0.0785; Orientation: 0.0070; Line Search: 0.0330
Low gradient: 5.287803498592567E-6
th(0)=1.3392379381061466E-9;dx=-2.7960865839727784E-11
Armijo: th(27.849533001676672)=1.3392379381061466E-9; dx=-2.8261946833026124E-11 evalInputDelta=0.0
Armijo: th(13.924766500838336)=1.3392379381061466E-9; dx=-2.8110801709509673E-11 evalInputDelta=0.0
Armijo: th(4.641588833612778)=1.3392379381061466E-9; dx=-2.801071089919988E-11 evalInputDelta=0.0
Armijo: th(1.1603972084031946)=1.3392379381061466E-9; dx=-2.797331462372768E-11 evalInputDelta=0.0
Armijo: th(0.23207944168063893)=1.3392379381061466E-9; dx=-2.7963354931637113E-11 evalInputDelta=0.0
Armijo: th(0.03867990694677315)=1.3392379381061466E-9; dx=-2.7961280665318817E-11 evalInputDelta=0.0
Armijo: th(0.005525700992396165)=1.3392379381061466E-9; dx=-2.7960925099928262E-11 evalInputDelta=0.0
Armijo: th(6.907126240495206E-4)=1.3392379381061466E-9; dx=-2.7960873247329155E-11 evalInputDelta=0.0
Armijo: th(7.67458471166134E-5)=1.3392379381061466E-9; dx=-2.796086666283935E-11 evalInputDelta=0.0
Armijo: th(7.67458471166134E-6)=1.3392379381061466E-9; dx=-2.7960865921913816E-11 evalInputDelta=0.0
Armijo: th(6.9768951924194E-7)=1.3392379381061466E-9; dx=-2.796086584714064E-11 evalInputDelta=0.0
Armijo: th(5.814079327016167E-8)=1.3392379381061466E-9; dx=-2.7960865840242917E-11 evalInputDelta=0.0
Armijo: th(4.472368713089359E-9)=1.3392379381061466E-9; dx=-2.796086583970369E-11 evalInputDelta=0.0
WOLFE (weak): th(3.1945490807781136E-10)=1.3392379381061466E-9; dx=-2.7960865839652607E-11 evalInputDelta=0.0
WOLFE (weak): th(2.3959118105835853E-9)=1.3392379381061466E-9; dx=-2.7960865839714946E-11 evalInputDelta=0.0
WOLFE (weak): th(3.4341402618364723E-9)=1.3392379381061466E-9; dx=-2.7960865839801186E-11 evalInputDelta=0.0
Armijo: th(3.9532544874629156E-9)=1.3392379381061466E-9; dx=-2.796086583979858E-11 evalInputDelta=0.0
WOLFE (weak): th(3.6936973746496937E-9)=1.3392379381061466E-9; dx=-2.7960865839707954E-11 evalInputDelta=0.0
Armijo: th(3.823475931056304E-9)=1.3392379381061466E-9; dx=-2.7960865839725587E-11 evalInputDelta=0.0
Armijo: th(3.758586652852999E-9)=1.3392379381061466E-9; dx=-2.7960865839799532E-11 evalInputDelta=0.0
Armijo: th(3.726142013751347E-9)=1.3392379381061466E-9; dx=-2.7960865839707954E-11 evalInputDelta=0.0
mu ~= nu (3.6936973746496937E-9): th(0.0)=1.3392379381061466E-9
Fitness changed from 1.3392379381061466E-9 to 1.3392379381061466E-9
Static Iteration Total: 0.1444; Orientation: 0.0011; Line Search: 0.1359
Iteration 2 failed. Error: 1.3392379381061466E-9
Previous Error: 0.0 -> 1.3392379381061466E-9
Optimization terminated 2
Final threshold in iteration 2: 1.3392379381061466E-9 (> 0.0) after 0.224s (< 30.000s)

Returns

    1.3392379381061466E-9

Training Converged

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:452 executed in 0.25 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 3724479861629
Reset training subject: 3724482879571
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=0.28015023395400235}, derivative=-0.0385718211016262}
New Minimum: 0.28015023395400235 > 0.280150233950145
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=0.280150233950145}, derivative=-0.03857182110178612}, evalInputDelta = -3.857358876757644E-12
New Minimum: 0.280150233950145 > 0.28015023392700195
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=0.28015023392700195}, derivative=-0.03857182110274565}, evalInputDelta = -2.7000401914278882E-11
New Minimum: 0.28015023392700195 > 0.2801502337650003
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=0.2801502337650003}, derivative=-0.03857182110946238}, evalInputDelta = -1.8900203624383494E-10
New Minimum: 0.2801502337650003 > 0.28015023263098876
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=0.28015023263098876}, derivative=-0.03857182115647944}, evalInputDelta = -1.3230135875730298E-9
New Minimum: 0.28015023263098876 > 0.28015022469290796
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=0.28015022469290796}, derivative=-0.0385718214855988}, evalInputDelta = -9.261094391366242E-9
New Minimum: 0.28015022469290796 > 0.2801501691263402
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=0.2801501691263402}, derivative=-0.03857182378943404}, evalInputDelta = -6.482766212734248E-8
New Minimum: 0.2801501691263402 > 0.2801497801602735
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=0.2801497801602735}, derivative=-0.03857183991625812}, evalInputDelta = -4.537937288717764E-7
New Minimum: 0.2801497801602735 > 0.28014705739325263
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=0.28014705739325263}, derivative=-0.03857195280292582}, evalInputDelta = -3.1765607497180604E-6
New Minimum: 0.28014705739325263 > 0.280127997800995
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=0.280127997800995}, derivative=-0.0385727429556528}, evalInputDelta = -2.2236153007337833E-5
New Minimum: 0.280127997800995 > 0.2799945697262833
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=0.2799945697262833}, derivative=-0.03857827137919136}, evalInputDelta = -1.5566422771906563E-4
New Minimum: 0.2799945697262833 > 0.2790600389087675
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.2790600389087675}, derivative=-0.038616839971652106}, evalInputDelta = -0.001090195045234832
New Minimum: 0.2790600389087675 > 0.27249257461236664
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.27249257461236664}, derivative=-0.03888017332944124}, evalInputDelta = -0.007657659341635703
New Minimum: 0.27249257461236664 > 0.2254405177166287
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.2254405177166287}, derivative=-0.04029746611308852}, evalInputDelta = -0.05470971623737364
New Minimum: 0.2254405177166287 > 7.23326584473997E-4
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=7.23326584473997E-4}, derivative=-0.0017364100126091245}, evalInputDelta = -0.27942690736952835
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=0.07102941658702915}, derivative=6.65201671967818E-4}, evalInputDelta = -0.2091208173669732
0.07102941658702915 <= 0.28015023395400235
F(66.6724873186157) = LineSearchPoint{point=PointSample{avg=0.07025216356606268}, derivative=6.869213038452626E-4}, evalInputDelta = -0.20989807038793967
Right bracket at 66.6724873186157
F(65.50589997747032) = LineSearchPoint{point=PointSample{avg=0.06943744757849994}, derivative=7.100117759822882E-4}, evalInputDelta = -0.2107127863755024
Right bracket at 65.50589997747032
F(64.32189309761738) = LineSearchPoint{point=PointSample{avg=0.06858235126209637}, derivative=7.34598363585804E-4}, evalInputDelta = -0.21156788269190596
Right bracket at 64.32189309761738
F(63.11978010805673) = LineSearchPoint{point=PointSample{avg=0.06768365088013593}, derivative=7.608208373503767E-4}, evalInputDelta = -0.2124665830738664
Right bracket at 63.11978010805673
F(61.898838884996) = LineSearchPoint{point=PointSample{avg=0.06673777733981494}, derivative=7.888353694008417E-4}, evalInputDelta = -0.2134124566141874
Right bracket at 61.898838884996
F(60.658310961553696) = LineSearchPoint{point=PointSample{avg=0.06574077181471422}, derivative=8.188166779670341E-4}, evalInputDelta = -0.21440946213928813
Right bracket at 60.658310961553696
F(59.39740127660409) = LineSearchPoint{point=PointSample{avg=0.064688235326653}, derivative=8.509604196133485E-4}, evalInputDelta = -0.21546199862734933
Right bracket at 59.39740127660409
F(58.11527871792442) = LineSearchPoint{point=PointSample{avg=0.06357527166359049}, derivative=8.854858224736609E-4}, evalInputDelta = -0.21657496229041184
Right bracket at 58.11527871792442
F(56.81107781356933) = LineSearchPoint{point=PointSample{avg=0.06239642310969749}, derivative=9.226385264362025E-4}, evalInputDelta = -0.21775381084430487
Right bracket at 56.81107781356933
F(55.483902062519924) = LineSearchPoint{point=PointSample{avg=0.06114559869741985}, derivative=9.626935490361875E-4}, evalInputDelta = -0.21900463525658248
Right bracket at 55.483902062519924
F(54.1328295866612) = LineSearchPoint{point=PointSample{avg=0.059815995149836376}, derivative=0.0010059582168895262}, evalInputDelta = -0.22033423880416597
Right bracket at 54.1328295866612
F(52.7569220530946) = LineSearchPoint{point=PointSample{avg=0.058400011510280916}, derivative=0.0010527747725744105}, evalInputDelta = -0.22175022244372145
Loops = 12
Fitness changed from 0.28015023395400235 to 7.23326584473997E-4
Iteration 1 complete. Error: 7.23326584473997E-4 Total: 0.1598; Orientation: 0.0006; Line Search: 0.1476
F(0.0) = LineSearchPoint{point=PointSample{avg=7.23326584473997E-4}, derivative=-1.88563906518136E-4}
New Minimum: 7.23326584473997E-4 > 4.706940402961747E-4
F(52.7569220530946) = LineSearchPoint{point=PointSample{avg=4.706940402961747E-4}, derivative=1.275714803442457E-4}, evalInputDelta = -2.526325441778223E-4
4.706940402961747E-4 <= 7.23326584473997E-4
New Minimum: 4.706940402961747E-4 > 1.3392379381061466E-9
F(31.46769305688277) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=1.0603867182594645E-8}, evalInputDelta = -7.233252452360589E-4
Right bracket at 31.46769305688277
Converged to right
Fitness changed from 7.23326584473997E-4 to 1.3392379381061466E-9
Iteration 2 complete. Error: 1.3392379381061466E-9 Total: 0.0138; Orientation: 0.0004; Line Search: 0.0108
Low gradient: 7.837789793961309E-7
F(0.0) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-6.143094885432406E-13}
F(31.46769305688277) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-6.14770997136411E-13}, evalInputDelta = 0.0
F(220.27385139817937) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-6.175510017448785E-13}, evalInputDelta = 0.0
F(1541.9169597872556) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-6.375499755757003E-13}, evalInputDelta = 0.0
F(10793.41871851079) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-8.093209925605253E-13}, evalInputDelta = 0.0
F(75553.93102957553) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-6.316803745564083E-11}, evalInputDelta = 0.0
F(528877.5172070287) = LineSearchPoint{point=PointSample{avg=4.5342364258412084E-4}, derivative=7.374685198602542E-9}, evalInputDelta = 4.5342230334618274E-4
F(40682.885939002204) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-2.3194322616995804E-12}, evalInputDelta = 0.0
F(284780.2015730154) = LineSearchPoint{point=PointSample{avg=6.867919984970566E-8}, derivative=5.480597366678086E-12}, evalInputDelta = 6.733996191159951E-8
F(21906.16935177042) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-1.1259203944120722E-12}, evalInputDelta = 0.0
F(153343.18546239292) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=8.929355195455758E-13}, evalInputDelta = 0.0
1.3392379381061466E-9 <= 1.3392379381061466E-9
F(62498.249008924904) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-9.493818794132007E-12}, evalInputDelta = 0.0
Left bracket at 62498.249008924904
F(145533.366525659) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=1.1332369654106947E-12}, evalInputDelta = 0.0
Right bracket at 145533.366525659
F(136678.7536652073) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=1.5447032141896092E-12}, evalInputDelta = 0.0
Right bracket at 136678.7536652073
F(126298.11902169407) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=2.3919262570779524E-12}, evalInputDelta = 0.0
Right bracket at 126298.11902169407
F(113458.82425100844) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=4.913095342006276E-12}, evalInputDelta = 0.0
Right bracket at 113458.82425100844
F(96080.07384811662) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=2.871344531293578E-11}, evalInputDelta = 0.0
Right bracket at 96080.07384811662
F(70842.72923410256) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-2.5628518799442452E-11}, evalInputDelta = 0.0
Left bracket at 70842.72923410256
F(82745.05502326073) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-3.7361016855743776E-9}, evalInputDelta = 0.0
Left bracket at 82745.05502326073
Converged to right
Fitness changed from 1.3392379381061466E-9 to 1.3392379381061466E-9
Static Iteration Total: 0.0772; Orientation: 0.0004; Line Search: 0.0746
Iteration 3 failed. Error: 1.3392379381061466E-9
Previous Error: 0.0 -> 1.3392379381061466E-9
Optimization terminated 3
Final threshold in iteration 3: 1.3392379381061466E-9 (> 0.0) after 0.251s (< 30.000s)

Returns

    1.3392379381061466E-9

Training Converged

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:509 executed in 0.12 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 3724736154991
Reset training subject: 3724738051692
Adding measurement 19472e20 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 0.28015023395400235 < 0.28015023395400235. Total: 1
th(0)=0.28015023395400235;dx=-0.0385718211016262
Adding measurement 5328e8b1 to history. Total: 1
New Minimum: 0.28015023395400235 > 0.1942286992847361
WOLFE (weak): th(2.154434690031884)=0.1942286992847361; dx=-0.04064599841669012 evalInputDelta=0.08592153466926625
Adding measurement 3bbe5786 to history. Total: 2
New Minimum: 0.1942286992847361 > 0.10877425370162033
WOLFE (weak): th(4.308869380063768)=0.10877425370162033; dx=-0.0374337486798352 evalInputDelta=0.17137598025238202
Adding measurement 4a8526aa to history. Total: 3
New Minimum: 0.10877425370162033 > 1.3392379381061466E-9
END: th(12.926608140191302)=1.3392379381061466E-9; dx=-5.540135503207175E-7 evalInputDelta=0.2801502326147644
Fitness changed from 0.28015023395400235 to 1.3392379381061466E-9
Iteration 1 complete. Error: 1.3392379381061466E-9 Total: 0.0253; Orientation: 0.0029; Line Search: 0.0163
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
Accepted: LBFGS Orientation magnitude: 1.892e-04, gradient 5.288e-06, dot 0.467; [cd8e9566-f56c-47f8-a890-b388d45c70b1 = 1.000/1.000e+00, 932bbec0-bc31-4acf-b9c2-91d923ab4e6b = 1.000/1.000e+00, f394fd03-ff70-430a-9410-90d5ef7c7016 = 1.000/1.000e+00, 047a6349-2624-4a51-a530-9df4fe0bb818 = 1.000/1.000e+00, 15a950c1-1100-43da-8056-162d78d052ed = 1.000/1.000e+00]
Constructing line search parameters: LBFGS
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
th(0)=1.3392379381061466E-9;dx=-4.667584174974566E-10
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
Armijo: th(2.154434690031884)=1.3392379381061466E-9; dx=-4.731498064860211E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
Armijo: th(1.077217345015942)=1.3392379381061466E-9; dx=-4.699378137884733E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
Armijo: th(0.3590724483386473)=1.3392379381061466E-9; dx=-4.678146217277426E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
Armijo: th(0.08976811208466183)=1.3392379381061466E-9; dx=-4.670221329631806E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
Armijo: th(0.017953622416932366)=1.3392379381061466E-9; dx=-4.668111427183505E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
Armijo: th(0.002992270402822061)=1.3392379381061466E-9; dx=-4.667672044137962E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
Armijo: th(4.2746720040315154E-4)=1.3392379381061466E-9; dx=-4.667596727566591E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
Armijo: th(5.343340005039394E-5)=1.3392379381061466E-9; dx=-4.667585744034735E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
Armijo: th(5.9370444500437714E-6)=1.3392379381061466E-9; dx=-4.667584349317751E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
Armijo: th(5.937044450043771E-7)=1.3392379381061466E-9; dx=-4.667584192408183E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
Armijo: th(5.397313136403428E-8)=1.3392379381061466E-9; dx=-4.667584176544096E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
Armijo: th(4.4977609470028565E-9)=1.3392379381061466E-9; dx=-4.667584175107617E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
Armijo: th(3.4598161130791205E-10)=1.3392379381061466E-9; dx=-4.667584174999907E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
WOLFE (weak): th(2.4712972236279432E-11)=1.3392379381061466E-9; dx=-4.667584174974111E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
WOLFE (weak): th(1.8534729177209574E-10)=1.3392379381061466E-9; dx=-4.667584174971698E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
Armijo: th(2.656644515400039E-10)=1.3392379381061466E-9; dx=-4.66758417498595E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
Armijo: th(2.255058716560498E-10)=1.3392379381061466E-9; dx=-4.667584174971297E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
WOLFE (weak): th(2.0542658171407276E-10)=1.3392379381061466E-9; dx=-4.667584174971512E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
WOLFE (weak): th(2.1546622668506128E-10)=1.3392379381061466E-9; dx=-4.667584174971326E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
WOLFE (weak): th(2.2048604917055554E-10)=1.3392379381061466E-9; dx=-4.667584174971157E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
Armijo: th(2.2299596041330265E-10)=1.3392379381061466E-9; dx=-4.667584174961862E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
Armijo: th(2.217410047919291E-10)=1.3392379381061466E-9; dx=-4.667584174968026E-10 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 4
mu ~= nu (2.2048604917055554E-10): th(0.0)=1.3392379381061466E-9
Fitness changed from 1.3392379381061466E-9 to 1.3392379381061466E-9
Static Iteration Total: 0.0945; Orientation: 0.0160; Line Search: 0.0762
Iteration 2 failed. Error: 1.3392379381061466E-9
Previous Error: 0.0 -> 1.3392379381061466E-9
Optimization terminated 2
Final threshold in iteration 2: 1.3392379381061466E-9 (> 0.0) after 0.120s (< 30.000s)

Returns

    1.3392379381061466E-9

Training Converged

TrainingTester.java:432 executed in 0.10 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging
Plotting range=[1.0, -8.8731422564261], [2.0, -3.1406655729532194]; valueStats=DoubleSummaryStatistics{count=4, sum=0.000723, min=0.000000, average=0.000181, max=0.000723}
Only 1 points for GD
Plotting 2 points for CjGD
Only 1 points for LBFGS

Returns

Result

TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging
Plotting range=[0.0, -8.8731422564261], [0.014, -3.1406655729532194]; valueStats=DoubleSummaryStatistics{count=4, sum=0.000723, min=0.000000, average=0.000181, max=0.000723}
Only 1 points for GD
Plotting 2 points for CjGD
Only 1 points for LBFGS

Returns

Result

Model Learning

In this apply, attempt to train a network to emulate a randomized network given an example input/output. The target state is:

TrainingTester.java:370 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> temp_18_0042 = network_target.state();
    assert temp_18_0042 != null;
    String temp_18_0041 = temp_18_0042.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b).orElse("");
    temp_18_0042.freeRef();
    return temp_18_0041;

Returns

    

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:480 executed in 0.11 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 3725123685235
Reset training subject: 3725125198386
Constructing line search parameters: GD
th(0)=0.2155825612615198;dx=-0.1917805627765573
Armijo: th(2.154434690031884)=0.34657359027997264; dx=0.0 evalInputDelta=-0.13099102901845283
Armijo: th(1.077217345015942)=0.34657359027997264; dx=0.0 evalInputDelta=-0.13099102901845283
Armijo: th(0.3590724483386473)=0.34657359027997264; dx=0.0 evalInputDelta=-0.13099102901845283
New Minimum: 0.2155825612615198 > 0.19369791039278908
WOLFE (weak): th(0.08976811208466183)=0.19369791039278908; dx=-0.3123628445540129 evalInputDelta=0.02188465086873073
New Minimum: 0.19369791039278908 > 0.11359409578643745
WOLFE (weak): th(0.22442028021165458)=0.11359409578643745; dx=-1.202634665319024 evalInputDelta=0.10198846547508236
Armijo: th(0.29174636427515094)=0.34657359027997264; dx=0.0 evalInputDelta=-0.13099102901845283
New Minimum: 0.11359409578643745 > 0.05687789885553389
WOLFE (weak): th(0.25808332224340275)=0.05687789885553389; dx=-2.364838308759182 evalInputDelta=0.1587046624059859
New Minimum: 0.05687789885553389 > 0.010406160449939564
WOLFE (weak): th(0.27491484325927684)=0.010406160449939564; dx=-2.7526255541709044 evalInputDelta=0.20517640081158026
Armijo: th(0.28333060376721386)=0.34657359027997264; dx=0.0 evalInputDelta=-0.13099102901845283
New Minimum: 0.010406160449939564 > 0.0012672871663876092
WOLFE (weak): th(0.27912272351324535)=0.0012672871663876092; dx=-1.2538939516883965 evalInputDelta=0.2143152740951322
New Minimum: 0.0012672871663876092 > 9.542363088151102E-6
END: th(0.28122666364022963)=9.542363088151102E-6; dx=-0.05682479728920844 evalInputDelta=0.21557301889843167
Fitness changed from 0.2155825612615198 to 9.542363088151102E-6
Iteration 1 complete. Error: 9.542363088151102E-6 Total: 0.0216; Orientation: 0.0002; Line Search: 0.0174
th(0)=9.542363088151102E-6;dx=-0.016837251597398798
Armijo: th(0.605884479908439)=0.34657359027997264; dx=0.0 evalInputDelta=-0.3465640479168845
Armijo: th(0.3029422399542195)=0.34657359027997264; dx=0.0 evalInputDelta=-0.3465640479168845
Armijo: th(0.1009807466514065)=0.34657359027997264; dx=0.0 evalInputDelta=-0.3465640479168845
Armijo: th(0.025245186662851624)=0.34657359027997264; dx=0.0 evalInputDelta=-0.3465640479168845
Armijo: th(0.005049037332570325)=0.34657359027997264; dx=0.0 evalInputDelta=-0.3465640479168845
New Minimum: 9.542363088151102E-6 > 1.4553673474223796E-6
END: th(8.415062220950542E-4)=1.4553673474223796E-6; dx=-0.004145179910946906 evalInputDelta=8.086995740728722E-6
Fitness changed from 9.542363088151102E-6 to 1.4553673474223796E-6
Iteration 2 complete. Error: 1.4553673474223796E-6 Total: 0.0156; Orientation: 0.0002; Line Search: 0.0132
th(0)=1.4553673474223796E-6;dx=-0.0010205060127967882
New Minimum: 1.4553673474223796E-6 > 3.24810987415317E-7
END: th(0.0018129701967592596)=3.24810987415317E-7; dx=-3.17978994244994E-4 evalInputDelta=1.1305563600070627E-6
Fitness changed from 1.4553673474223796E-6 to 3.24810987415317E-7
Iteration 3 complete. Error: 3.24810987415317E-7 Total: 0.0052; Orientation: 0.0002; Line Search: 0.0033
th(0)=3.24810987415317E-7;dx=-9.907892703537844E-5
New Minimum: 3.24810987415317E-7 > 8.292323826884735E-8
END: th(0.003905925883892079)=8.292323826884735E-8; dx=-3.3285930181983885E-5 evalInputDelta=2.4188774914646967E-7
Fitness changed from 3.24810987415317E-7 to 8.292323826884735E-8
Iteration 4 complete. Error: 8.292323826884735E-8 Total: 0.0066; Orientation: 0.0003; Line Search: 0.0042
th(0)=8.292323826884735E-8;dx=-1.1182530748281978E-5
New Minimum: 8.292323826884735E-8 > 2.315471440463034E-8
END: th(0.008415062220950544)=2.315471440463034E-8; dx=-3.952202780636919E-6 evalInputDelta=5.976852386421701E-8
Fitness changed from 8.292323826884735E-8 to 2.315471440463034E-8
Iteration 5 complete. Error: 2.315471440463034E-8 Total: 0.0053; Orientation: 0.0002; Line Search: 0.0035
th(0)=2.315471440463034E-8;dx=-1.3968132233102916E-6
New Minimum: 2.315471440463034E-8 > 6.884964090732676E-9
END: th(0.0181297019675926)=6.884964090732676E-9; dx=-5.114037033389238E-7 evalInputDelta=1.6269750313897664E-8
Fitness changed from 2.315471440463034E-8 to 6.884964090732676E-9
Iteration 6 complete. Error: 6.884964090732676E-9 Total: 0.0049; Orientation: 0.0002; Line Search: 0.0032
th(0)=6.884964090732676E-9;dx=-1.8723601940777748E-7
New Minimum: 6.884964090732676E-9 > 2.1456233502655176E-9
END: th(0.0390592588389208)=2.1456233502655176E-9; dx=-7.036726441871025E-8 evalInputDelta=4.7393407404671576E-9
Fitness changed from 6.884964090732676E-9 to 2.1456233502655176E-9
Iteration 7 complete. Error: 2.1456233502655176E-9 Total: 0.0050; Orientation: 0.0002; Line Search: 0.0034
th(0)=2.1456233502655176E-9;dx=-2.6445509349292478E-8
New Minimum: 2.1456233502655176E-9 > 1.0305768471459636E-9
END: th(0.08415062220950545)=1.0305768471459636E-9; dx=-1.5069112183524385E-8 evalInputDelta=1.115046503119554E-9
Fitness changed from 2.1456233502655176E-9 to 1.0305768471459636E-9
Iteration 8 complete. Error: 1.0305768471459636E-9 Total: 0.0064; Orientation: 0.0001; Line Search: 0.0042
th(0)=1.0305768471459636E-9;dx=-8.58664278310519E-9
Armijo: th(0.18129701967592604)=1.0305768471459636E-9; dx=-1.092087636680392E-8 evalInputDelta=0.0
Armijo: th(0.09064850983796302)=1.0305768471459636E-9; dx=-9.63717892525209E-9 evalInputDelta=0.0
Armijo: th(0.030216169945987675)=1.0305768471459636E-9; dx=-8.914713807255796E-9 evalInputDelta=0.0
Armijo: th(0.007554042486496919)=1.0305768471459636E-9; dx=-8.666752545316509E-9 evalInputDelta=0.0
Armijo: th(0.0015108084972993837)=1.0305768471459636E-9; dx=-8.602565760416744E-9 evalInputDelta=0.0
Armijo: th(2.5180141621656394E-4)=1.0305768471459636E-9; dx=-8.589293199938497E-9 evalInputDelta=0.0
Armijo: th(3.5971630888080565E-5)=1.0305768471459636E-9; dx=-8.587021330622543E-9 evalInputDelta=0.0
Armijo: th(4.496453861010071E-6)=1.0305768471459636E-9; dx=-8.58669010002343E-9 evalInputDelta=0.0
Armijo: th(4.996059845566746E-7)=1.0305768471459636E-9; dx=-8.586648040519437E-9 evalInputDelta=0.0
Armijo: th(4.996059845566745E-8)=1.0305768471459636E-9; dx=-8.586643308848578E-9 evalInputDelta=0.0
Armijo: th(4.541872586878859E-9)=1.0305768471459636E-9; dx=-8.586642830898896E-9 evalInputDelta=0.0
Armijo: th(3.7848938223990494E-10)=1.0305768471459636E-9; dx=-8.586642787088E-9 evalInputDelta=0.0
Armijo: th(2.911456786460807E-11)=1.0305768471459636E-9; dx=-8.58664278341374E-9 evalInputDelta=0.0
WOLFE (weak): th(2.0796119903291477E-12)=1.0305768471459636E-9; dx=-8.58664278312723E-9 evalInputDelta=0.0
Armijo: th(1.559708992746861E-11)=1.0305768471459636E-9; dx=-8.586642783268905E-9 evalInputDelta=0.0
WOLFE (weak): th(8.838350958898878E-12)=1.0305768471459636E-9; dx=-8.58664278319964E-9 evalInputDelta=0.0
Armijo: th(1.2217720443183744E-11)=1.0305768471459636E-9; dx=-8.586642783233646E-9 evalInputDelta=0.0
WOLFE (weak): th(1.0528035701041312E-11)=1.0305768471459636E-9; dx=-8.586642783215383E-9 evalInputDelta=0.0
WOLFE (weak): th(1.1372878072112529E-11)=1.0305768471459636E-9; dx=-8.586642783224833E-9 evalInputDelta=0.0
WOLFE (weak): th(1.1795299257648137E-11)=1.0305768471459636E-9; dx=-8.586642783231126E-9 evalInputDelta=0.0
WOLFE (weak): th(1.2006509850415942E-11)=1.0305768471459636E-9; dx=-8.586642783231126E-9 evalInputDelta=0.0
Armijo: th(1.2112115146799844E-11)=1.0305768471459636E-9; dx=-8.586642783233646E-9 evalInputDelta=0.0
mu ~= nu (1.2006509850415942E-11): th(0.0)=1.0305768471459636E-9
Fitness changed from 1.0305768471459636E-9 to 1.0305768471459636E-9
Static Iteration Total: 0.0414; Orientation: 0.0003; Line Search: 0.0389
Iteration 9 failed. Error: 1.0305768471459636E-9
Previous Error: 0.0 -> 1.0305768471459636E-9
Optimization terminated 9
Final threshold in iteration 9: 1.0305768471459636E-9 (> 0.0) after 0.112s (< 30.000s)

Returns

    1.0305768471459636E-9

Training Converged

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:452 executed in 0.20 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 3725240717920
Reset training subject: 3725242023121
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=0.2155825612615198}, derivative=-0.1917805627765573}
New Minimum: 0.2155825612615198 > 0.2155825612423417
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=0.2155825612423417}, derivative=-0.19178056286487194}, evalInputDelta = -1.9178103549677417E-11
New Minimum: 0.2155825612423417 > 0.2155825611272734
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=0.2155825611272734}, derivative=-0.19178056339475893}, evalInputDelta = -1.3424641953641014E-10
New Minimum: 0.2155825611272734 > 0.21558256032179499
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=0.21558256032179499}, derivative=-0.19178056710396835}, evalInputDelta = -9.397248257325685E-10
New Minimum: 0.21558256032179499 > 0.21558255468344592
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=0.21558255468344592}, derivative=-0.19178059306843656}, evalInputDelta = -6.578073891150282E-9
New Minimum: 0.21558255468344592 > 0.2155825152149812
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=0.2155825152149812}, derivative=-0.19178077481988254}, evalInputDelta = -4.60465386098452E-8
New Minimum: 0.2155825152149812 > 0.21558223893468056
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=0.21558223893468056}, derivative=-0.19178204708819027}, evalInputDelta = -3.2232683924715566E-7
New Minimum: 0.21558223893468056 > 0.21558030492125554
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=0.21558030492125554}, derivative=-0.19179095336748397}, evalInputDelta = -2.2563402642672425E-6
New Minimum: 0.21558030492125554 > 0.2155667643120293
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=0.2155667643120293}, derivative=-0.1918533169843509}, evalInputDelta = -1.579694949049859E-5
New Minimum: 0.2155667643120293 > 0.21547185661526047
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=0.21547185661526047}, derivative=-0.1922908277751144}, evalInputDelta = -1.1070464625934084E-4
New Minimum: 0.21547185661526047 > 0.21480139043522856
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=0.21480139043522856}, derivative=-0.1954014228068702}, evalInputDelta = -7.811708262912553E-4
New Minimum: 0.21480139043522856 > 0.20978476166384552
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.20978476166384552}, derivative=-0.21979984046150752}, evalInputDelta = -0.005797799597674291
New Minimum: 0.20978476166384552 > 0.13998982213177694
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.13998982213177694}, derivative=-0.8171609174286483}, evalInputDelta = -0.07559273912974288
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.13099102901845283
F(0.10647144000769232) = LineSearchPoint{point=PointSample{avg=0.18817965276249837}, derivative=-0.3495611331231546}, evalInputDelta = -0.02740290849902144
F(0.7453000800538463) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.13099102901845283
F(0.05733077538875741) = LineSearchPoint{point=PointSample{avg=0.20287837455186483}, derivative=-0.2567655518597191}, evalInputDelta = -0.01270418670965498
F(0.40131542772130185) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.13099102901845283
F(0.03087041751702322) = LineSearchPoint{point=PointSample{avg=0.20920439196458135}, derivative=-0.22275303281409314}, evalInputDelta = -0.006378169296938463
New Minimum: 0.13998982213177694 > 0.1229674672165613
F(0.21609292261916255) = LineSearchPoint{point=PointSample{avg=0.1229674672165613}, derivative=-1.054052622872364}, evalInputDelta = -0.09261509404495852
F(1.5126504583341378) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.13099102901845283
F(0.11635772756416445) = LineSearchPoint{point=PointSample{avg=0.18459914694151308}, derivative=-0.37529511429587964}, evalInputDelta = -0.03098341432000673
F(0.8145040929491512) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.13099102901845283
F(0.06265416099608855) = LineSearchPoint{point=PointSample{avg=0.201490614483517}, derivative=-0.2646851102371645}, evalInputDelta = -0.014091946778002823
F(0.4385791269726198) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.13099102901845283
F(0.033736855920970754) = LineSearchPoint{point=PointSample{avg=0.20856116666374377}, derivative=-0.2260583241136129}, evalInputDelta = -0.007021394597776037
New Minimum: 0.1229674672165613 > 0.09794016932619318
F(0.23615799144679528) = LineSearchPoint{point=PointSample{avg=0.09794016932619318}, derivative=-1.4804879455314897}, evalInputDelta = -0.11764239193532663
F(1.653105940127567) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.13099102901845283
F(0.12716199539442824) = LineSearchPoint{point=PointSample{avg=0.18037532573932427}, derivative=-0.4073293372547843}, evalInputDelta = -0.03520723552219554
F(0.8901339677609976) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.13099102901845283
F(0.0684718436739229) = LineSearchPoint{point=PointSample{avg=0.19992442545550615}, derivative=-0.27382650277267606}, evalInputDelta = -0.015658135806013662
F(0.47930290571746026) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.13099102901845283
F(0.03686945428595848) = LineSearchPoint{point=PointSample{avg=0.20784723474034122}, derivative=-0.2297669057992589}, evalInputDelta = -0.007735326521178587
New Minimum: 0.09794016932619318 > 0.05687114049616375
F(0.25808618000170935) = LineSearchPoint{point=PointSample{avg=0.05687114049616375}, derivative=-2.3649944114046795}, evalInputDelta = -0.15871142076535605
F(1.8066032600119655) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.13099102901845283
F(0.13896948153938196) = LineSearchPoint{point=PointSample{avg=0.1753319483650057}, derivative=-0.44803855122779496}, evalInputDelta = -0.040250612896514104
F(0.9727863707756738) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.13099102901845283
F(0.07482972082889798) = LineSearchPoint{point=PointSample{avg=0.19815007104243337}, derivative=-0.2844474174503269}, evalInputDelta = -0.01743249021908644
F(0.5238080458022859) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.13099102901845283
F(0.04029292660017584) = LineSearchPoint{point=PointSample{avg=0.20705352801187246}, derivative=-0.23393956144194009}, evalInputDelta = -0.00852903324964735
New Minimum: 0.05687114049616375 > 1.0305768471459636E-9
F(0.28205048620123085) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-2.0293640811592863E-4}, evalInputDelta = -0.21558256023094297
F(1.974353403408616) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.13099102901845283
F(0.1518733387237397) = LineSearchPoint{point=PointSample{avg=0.16921895709400317}, derivative=-0.5011144499187401}, evalInputDelta = -0.04636360416751664
F(1.063113371066178) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.13099102901845283
F(0.08177795162047523) = LineSearchPoint{point=PointSample{avg=0.19613099477726498}, derivative=-0.2968804213101723}, evalInputDelta = -0.019451566484254834
F(0.5724456613433266) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.13099102901845283
F(0.044034281641794355) = LineSearchPoint{point=PointSample{avg=0.20616951695327507}, derivative=-0.23864899047504137}, evalInputDelta = -0.009413044308244745
F(0.3082399714925605) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.13099102901845283
F(0.02371076703788927) = LineS

...skipping 2122 bytes...

derivative=-9.139064931640039E-7}, evalInputDelta = 0.0
F(0.3627736564458853) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.027905665880452714) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-3.9458648807982E-7}, evalInputDelta = 0.0
F(0.195339661163169) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.015026127781782231) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-2.889526313533382E-7}, evalInputDelta = 0.0
F(0.10518289447247561) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.008090991882498124) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-2.5011994370235276E-7}, evalInputDelta = 0.0
F(0.05663694317748687) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-1.1682557896723863E-6}, evalInputDelta = 0.0
F(0.3964586022424081) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.030496815557108314) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-4.2376432883046435E-7}, evalInputDelta = 0.0
F(0.2134777088997582) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.016421362223058324) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-2.979884130547774E-7}, evalInputDelta = 0.0
F(0.11494953556140827) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.008842271966262175) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-2.538950300050677E-7}, evalInputDelta = 0.0
F(0.061895903763835226) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-1.615591703351381E-6}, evalInputDelta = 0.0
F(0.43327132634684656) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.03332856356514204) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-4.6000075695691253E-7}, evalInputDelta = 0.0
F(0.2332999449559943) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.01794614961199956) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-3.0841447432468946E-7}, evalInputDelta = 0.0
F(0.12562304728399692) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.009663311329538226) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-2.581304707409789E-7}, evalInputDelta = 0.0
F(0.06764317930676758) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-2.556906629721044E-6}, evalInputDelta = 0.0
F(0.4735022551473731) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.036423250395951776) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-5.059048180517022E-7}, evalInputDelta = 0.0
F(0.2549627527716624) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.019612519443974033) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-3.2052234686447304E-7}, evalInputDelta = 0.0
F(0.13728763610781825) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.010560587392909096) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-2.6289553579578746E-7}, evalInputDelta = 0.0
F(0.07392411175036367) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-5.38430235545453E-6}, evalInputDelta = 0.0
F(0.5174687822525457) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.03980529094250351) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-5.654946043855785E-7}, evalInputDelta = 0.0
F(0.2786370365975246) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.021433618199809584) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-3.3468749483534977E-7}, evalInputDelta = 0.0
F(0.1500353273986671) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.0115411790306667) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-2.6827302826260915E-7}, evalInputDelta = 0.0
F(0.0807882532146669) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-3.319677530599746E-5}, evalInputDelta = 0.0
F(0.5655177725026683) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.04350136711558987) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-6.452650656799859E-7}, evalInputDelta = 0.0
F(0.3045095698091291) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.023423813062240697) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-3.514013199490921E-7}, evalInputDelta = 0.0
F(0.16396669143568487) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.012612822418129607) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-2.743627074486787E-7}, evalInputDelta = 0.0
F(0.08828975692690724) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.00679151976360825) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-2.43807127862592E-7}, evalInputDelta = 0.0
F(0.04754063834525775) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-7.563712703710011E-7}, evalInputDelta = 0.0
F(0.33278446841680426) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.025598805262831096) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-3.713187900191333E-7}, evalInputDelta = 0.0
F(0.17919163683981768) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.01378397206460136) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-2.812857981859265E-7}, evalInputDelta = 0.0
F(0.09648780445220952) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
F(0.007422138804016117) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-2.468370858356831E-7}, evalInputDelta = 0.0
Loops = 52
F(0.014844277608032233) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-2.8780858611988787E-7}, evalInputDelta = 0.0
Right bracket at 0.014844277608032233
F(0.029688555216064467) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-4.142867316103892E-7}, evalInputDelta = 0.0
Right bracket at 0.029688555216064467
F(0.059377110432128934) = LineSearchPoint{point=PointSample{avg=1.0305768471459636E-9}, derivative=-1.3711881808778122E-6}, evalInputDelta = 0.0
Right bracket at 0.059377110432128934
F(0.11875422086425787) = LineSearchPoint{point=PointSample{avg=0.34657359027997264}, derivative=0.0}, evalInputDelta = 0.3465735892493958
0.34657359027997264 > 1.0305768471459636E-9
Fitness changed from 1.0305768471459636E-9 to 1.0305768471459636E-9
Static Iteration Total: 0.1007; Orientation: 0.0002; Line Search: 0.0991
Iteration 2 failed. Error: 1.0305768471459636E-9
Previous Error: 0.0 -> 1.0305768471459636E-9
Optimization terminated 2
Final threshold in iteration 2: 1.0305768471459636E-9 (> 0.0) after 0.202s (< 30.000s)

Returns

    1.0305768471459636E-9

Training Converged

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:509 executed in 0.09 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 3725450271048
Reset training subject: 3725451571194
Adding measurement 39e3630c to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 0.2155825612615198 < 0.2155825612615198. Total: 1
th(0)=0.2155825612615198;dx=-0.1917805627765573
Non-optimal measurement 0.34657359027997264 < 0.2155825612615198. Total: 1
Armijo: th(2.154434690031884)=0.34657359027997264; dx=0.0 evalInputDelta=-0.13099102901845283
Non-optimal measurement 0.34657359027997264 < 0.2155825612615198. Total: 1
Armijo: th(1.077217345015942)=0.34657359027997264; dx=0.0 evalInputDelta=-0.13099102901845283
Non-optimal measurement 0.34657359027997264 < 0.2155825612615198. Total: 1
Armijo: th(0.3590724483386473)=0.34657359027997264; dx=0.0 evalInputDelta=-0.13099102901845283
Adding measurement 2a6806f2 to history. Total: 1
New Minimum: 0.2155825612615198 > 0.19369791039278908
WOLFE (weak): th(0.08976811208466183)=0.19369791039278908; dx=-0.3123628445540129 evalInputDelta=0.02188465086873073
Adding measurement 640dd08e to history. Total: 2
New Minimum: 0.19369791039278908 > 0.11359409578643745
WOLFE (weak): th(0.22442028021165458)=0.11359409578643745; dx=-1.202634665319024 evalInputDelta=0.10198846547508236
Non-optimal measurement 0.34657359027997264 < 0.11359409578643745. Total: 3
Armijo: th(0.29174636427515094)=0.34657359027997264; dx=0.0 evalInputDelta=-0.13099102901845283
Adding measurement 693aeb3e to history. Total: 3
New Minimum: 0.11359409578643745 > 0.05687789885553389
WOLFE (weak): th(0.25808332224340275)=0.05687789885553389; dx=-2.364838308759182 evalInputDelta=0.1587046624059859
Adding measurement 19948b68 to history. Total: 4
New Minimum: 0.05687789885553389 > 0.010406160449939564
WOLFE (weak): th(0.27491484325927684)=0.010406160449939564; dx=-2.7526255541709044 evalInputDelta=0.20517640081158026
Non-optimal measurement 0.34657359027997264 < 0.010406160449939564. Total: 5
Armijo: th(0.28333060376721386)=0.34657359027997264; dx=0.0 evalInputDelta=-0.13099102901845283
Adding measurement 5eec866f to history. Total: 5
New Minimum: 0.010406160449939564 > 0.0012672871663876092
WOLFE (weak): th(0.27912272351324535)=0.0012672871663876092; dx=-1.2538939516883965 evalInputDelta=0.2143152740951322
Adding measurement 792b4630 to history. Total: 6
New Minimum: 0.0012672871663876092 > 9.542363088151102E-6
END: th(0.28122666364022963)=9.542363088151102E-6; dx=-0.05682479728920844 evalInputDelta=0.21557301889843167
Fitness changed from 0.2155825612615198 to 9.542363088151102E-6
Iteration 1 complete. Error: 9.542363088151102E-6 Total: 0.0239; Orientation: 0.0006; Line Search: 0.0196
Non-optimal measurement 9.542363088151102E-6 < 9.542363088151102E-6. Total: 7
Accepted: LBFGS Orientation magnitude: 1.853e-02, gradient 1.298e-01, dot 1.000; [e61ac1c1-5af1-4ace-bd81-7040a0529dbc = 1.000/1.000e+00]
Constructing line search parameters: LBFGS
Non-optimal measurement 9.542363088151102E-6 < 9.542363088151102E-6. Total: 7
th(0)=9.542363088151102E-6;dx=-0.002403888855984594
Non-optimal measurement 0.34657359027997264 < 9.542363088151102E-6. Total: 7
Armijo: th(2.154434690031884)=0.34657359027997264; dx=0.0 evalInputDelta=-0.3465640479168845
Non-optimal measurement 0.34657359027997264 < 9.542363088151102E-6. Total: 7
Armijo: th(1.077217345015942)=0.34657359027997264; dx=0.0 evalInputDelta=-0.3465640479168845
Non-optimal measurement 0.34657359027997264 < 9.542363088151102E-6. Total: 7
Armijo: th(0.3590724483386473)=0.34657359027997264; dx=0.0 evalInputDelta=-0.3465640479168845
Non-optimal measurement 0.34657359027997264 < 9.542363088151102E-6. Total: 7
Armijo: th(0.08976811208466183)=0.34657359027997264; dx=0.0 evalInputDelta=-0.3465640479168845
Adding measurement 7fc4c3d6 to history. Total: 7
New Minimum: 9.542363088151102E-6 > 1.0305768471459636E-9
END: th(0.017953622416932366)=1.0305768471459636E-9; dx=-3.785028370008216E-6 evalInputDelta=9.541332511303956E-6
Fitness changed from 9.542363088151102E-6 to 1.0305768471459636E-9
Iteration 2 complete. Error: 1.0305768471459636E-9 Total: 0.0165; Orientation: 0.0058; Line Search: 0.0093
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
Accepted: LBFGS Orientation magnitude: 2.917e-05, gradient 2.043e-04, dot 1.000; [e61ac1c1-5af1-4ace-bd81-7040a0529dbc = 1.000/1.000e+00]
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
th(0)=1.0305768471459636E-9;dx=-5.959693072373421E-9
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
Armijo: th(0.03867990694677316)=1.0305768471459636E-9; dx=-6.113580871791114E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
Armijo: th(0.01933995347338658)=1.0305768471459636E-9; dx=-6.035819673818453E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
Armijo: th(0.006446651157795527)=1.0305768471459636E-9; dx=-5.9848899663186795E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
Armijo: th(0.0016116627894488817)=1.0305768471459636E-9; dx=-5.965975699334884E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
Armijo: th(3.2233255788977636E-4)=1.0305768471459636E-9; dx=-5.960948715372505E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
Armijo: th(5.372209298162939E-5)=1.0305768471459636E-9; dx=-5.9599023155934725E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
Armijo: th(7.674584711661342E-6)=1.0305768471459636E-9; dx=-5.959722963511322E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
Armijo: th(9.593230889576677E-7)=1.0305768471459636E-9; dx=-5.9596968087501426E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
Armijo: th(1.0659145432862975E-7)=1.0305768471459636E-9; dx=-5.959693487528253E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
Armijo: th(1.0659145432862976E-8)=1.0305768471459636E-9; dx=-5.959693113889643E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
Armijo: th(9.690132211693615E-10)=1.0305768471459636E-9; dx=-5.959693076149306E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
Armijo: th(8.075110176411346E-11)=1.0305768471459636E-9; dx=-5.9596930726880784E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
WOLFE (weak): th(6.211623212624112E-12)=1.0305768471459636E-9; dx=-5.959693072395633E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
Armijo: th(4.3481362488368783E-11)=1.0305768471459636E-9; dx=-5.959693072542966E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
Armijo: th(2.4846492850496447E-11)=1.0305768471459636E-9; dx=-5.959693072469668E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
WOLFE (weak): th(1.552905803156028E-11)=1.0305768471459636E-9; dx=-5.959693072432652E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
Armijo: th(2.0187775441028364E-11)=1.0305768471459636E-9; dx=-5.9596930724511615E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
Armijo: th(1.7858416736294324E-11)=1.0305768471459636E-9; dx=-5.959693072440054E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
WOLFE (weak): th(1.66937373839273E-11)=1.0305768471459636E-9; dx=-5.959693072439315E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
WOLFE (weak): th(1.7276077060110812E-11)=1.0305768471459636E-9; dx=-5.959693072440054E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
Armijo: th(1.756724689820257E-11)=1.0305768471459636E-9; dx=-5.959693072440054E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
Armijo: th(1.742166197915669E-11)=1.0305768471459636E-9; dx=-5.959693072440054E-9 evalInputDelta=0.0
Non-optimal measurement 1.0305768471459636E-9 < 1.0305768471459636E-9. Total: 8
mu ~= nu (1.7276077060110812E-11): th(0.0)=1.0305768471459636E-9
Fitness changed from 1.0305768471459636E-9 to 1.0305768471459636E-9
Static Iteration Total: 0.0541; Orientation: 0.0063; Line Search: 0.0464
Iteration 3 failed. Error: 1.0305768471459636E-9
Previous Error: 0.0 -> 1.0305768471459636E-9
Optimization terminated 3
Final threshold in iteration 3: 1.0305768471459636E-9 (> 0.0) after 0.095s (< 30.000s)

Returns

    1.0305768471459636E-9

Training Converged

TrainingTester.java:432 executed in 0.01 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging
Plotting range=[1.0, -8.986919618589186], [8.0, -5.020344062501425]; valueStats=DoubleSummaryStatistics{count=11, sum=0.000021, min=0.000000, average=0.000002, max=0.000010}
Plotting 8 points for GD
Only 1 points for CjGD
Plotting 2 points for LBFGS

Returns

Result

TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging
Plotting range=[0.0, -8.986919618589186], [0.049, -5.020344062501425]; valueStats=DoubleSummaryStatistics{count=11, sum=0.000021, min=0.000000, average=0.000002, max=0.000010}
Plotting 8 points for GD
Only 1 points for CjGD
Plotting 2 points for LBFGS

Returns

Result

Composite Learning

In this apply, attempt to train a network to emulate a randomized network given an example input/output. The target state is:

TrainingTester.java:279 executed in 0.00 seconds (0.000 gc):

    RefList<double[]> temp_18_0037 = temp_18_0035.state();
    assert temp_18_0037 != null;
    String temp_18_0036 = temp_18_0037.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b).orElse("");
    temp_18_0037.freeRef();
    return temp_18_0036;

Returns

    

We simultaneously regress this target input:

TrainingTester.java:287 executed in 0.00 seconds (0.000 gc):

    return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(x -> {
      RefStream<Tensor> temp_18_0006 = RefArrays.stream(RefUtil.addRef(x));
      if (null != x)
        RefUtil.freeRef(x);
      return temp_18_0006;
    }).map(x -> {
      String temp_18_0007 = x.prettyPrint();
      x.freeRef();
      return temp_18_0007;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [ 0.7, -0.128, 0.08 ]
    [ -0.128, 0.7, 0.08 ]
    [ -0.128, 0.08, 0.7 ]
    [ 0.08, -0.128, 0.7 ]
    [ 0.7, -0.128, 0.08 ]

Which produces the following output:

TrainingTester.java:308 executed in 0.00 seconds (0.000 gc):

    return RefStream.of(RefUtil.addRef(output_target)).map(x -> {
      String temp_18_0008 = x.prettyPrint();
      x.freeRef();
      return temp_18_0008;
    }).reduce((a, b) -> a + "\n" + b).orElse("");

Returns

    [ 0.649751613290642 ]
    [ 0.649751613290642 ]
    [ 0.649751613290642 ]
    [ 0.649751613290642 ]
    [ 0.649751613290642 ]

Gradient Descent

First, we train using basic gradient descent method apply weak line search conditions.

TrainingTester.java:480 executed in 0.12 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 3725676684591
Reset training subject: 3725678762705
Constructing line search parameters: GD
th(0)=0.28015023395400235;dx=-0.36243331365923415
Armijo: th(2.154434690031884)=0.4503734988166844; dx=0.0 evalInputDelta=-0.17022326486268208
Armijo: th(1.077217345015942)=0.4503734988166844; dx=0.0 evalInputDelta=-0.17022326486268208
Armijo: th(0.3590724483386473)=0.4503734988166844; dx=0.0 evalInputDelta=-0.17022326486268208
New Minimum: 0.28015023395400235 > 0.2349583540168329
WOLFE (weak): th(0.08976811208466183)=0.2349583540168329; dx=-0.7121026059724442 evalInputDelta=0.045191879937169444
Armijo: th(0.22442028021165458)=0.4503734988166844; dx=0.0 evalInputDelta=-0.17022326486268208
New Minimum: 0.2349583540168329 > 0.16114929598092997
WOLFE (weak): th(0.1570941961481582)=0.16114929598092997; dx=-1.7379704823916264 evalInputDelta=0.11900093797307237
New Minimum: 0.16114929598092997 > 0.07446957176321886
WOLFE (weak): th(0.1907572381799064)=0.07446957176321886; dx=-3.8413492907915505 evalInputDelta=0.2056806621907835
New Minimum: 0.07446957176321886 > 0.002039857293237384
WOLFE (weak): th(0.2075887591957805)=0.002039857293237384; dx=-2.2408747063251764 evalInputDelta=0.278110376660765
Armijo: th(0.21600451970371753)=0.4503734988166844; dx=0.0 evalInputDelta=-0.17022326486268208
Armijo: th(0.21179663944974902)=0.4503734988166844; dx=0.0 evalInputDelta=-0.17022326486268208
New Minimum: 0.002039857293237384 > 1.502688973835706E-6
END: th(0.20969269932276474)=1.502688973835706E-6; dx=-0.018856070265211754 evalInputDelta=0.28014873126502854
Fitness changed from 0.28015023395400235 to 1.502688973835706E-6
Iteration 1 complete. Error: 1.502688973835706E-6 Total: 0.0321; Orientation: 0.0003; Line Search: 0.0261
th(0)=1.502688973835706E-6;dx=-0.0010620802050603443
Armijo: th(0.4517692256673897)=0.4503734988166844; dx=0.0 evalInputDelta=-0.4503719961277106
Armijo: th(0.22588461283369485)=0.4503734988166844; dx=0.0 evalInputDelta=-0.4503719961277106
Armijo: th(0.07529487094456495)=0.4503734988166844; dx=0.0 evalInputDelta=-0.4503719961277106
Armijo: th(0.01882371773614124)=0.4503734988166844; dx=0.0 evalInputDelta=-0.4503719961277106
New Minimum: 1.502688973835706E-6 > 2.9847984121548896E-8
END: th(0.0037647435472282476)=2.9847984121548896E-8; dx=-4.656095004852672E-5 evalInputDelta=1.472840989714157E-6
Fitness changed from 1.502688973835706E-6 to 2.9847984121548896E-8
Iteration 2 complete. Error: 2.9847984121548896E-8 Total: 0.0130; Orientation: 0.0003; Line Search: 0.0111
th(0)=2.9847984121548896E-8;dx=-2.0412038827751634E-6
New Minimum: 2.9847984121548896E-8 > 1.6652168805180976E-8
END: th(0.008110894097222224)=1.6652168805180976E-8; dx=-1.2609075103063415E-6 evalInputDelta=1.319581531636792E-8
Fitness changed from 2.9847984121548896E-8 to 1.6652168805180976E-8
Iteration 3 complete. Error: 1.6652168805180976E-8 Total: 0.0061; Orientation: 0.0003; Line Search: 0.0042
th(0)=1.6652168805180976E-8;dx=-7.788970832696881E-7
New Minimum: 1.6652168805180976E-8 > 6.910119187141267E-9
END: th(0.017474391610230396)=6.910119187141267E-9; dx=-3.745056682281273E-7 evalInputDelta=9.742049618039709E-9
Fitness changed from 1.6652168805180976E-8 to 6.910119187141267E-9
Iteration 4 complete. Error: 6.910119187141267E-9 Total: 0.0065; Orientation: 0.0003; Line Search: 0.0043
th(0)=6.910119187141267E-9;dx=-1.8006807153384735E-7
New Minimum: 6.910119187141267E-9 > 2.3763957438565355E-9
END: th(0.03764743547228248)=2.3763957438565355E-9; dx=-7.33667842686627E-8 evalInputDelta=4.533723443284731E-9
Fitness changed from 6.910119187141267E-9 to 2.3763957438565355E-9
Iteration 5 complete. Error: 2.3763957438565355E-9 Total: 0.0067; Orientation: 0.0003; Line Search: 0.0045
th(0)=2.3763957438565355E-9;dx=-2.989250118128484E-8
New Minimum: 2.3763957438565355E-9 > 1.3392379381061466E-9
END: th(0.08110894097222225)=1.3392379381061466E-9; dx=-1.9897312299955165E-8 evalInputDelta=1.0371578057503889E-9
Fitness changed from 2.3763957438565355E-9 to 1.3392379381061466E-9
Iteration 6 complete. Error: 1.3392379381061466E-9 Total: 0.0063; Orientation: 0.0003; Line Search: 0.0039
th(0)=1.3392379381061466E-9;dx=-1.3244225859993263E-8
Armijo: th(0.174743916102304)=1.3392379381061466E-9; dx=-1.7461589229560516E-8 evalInputDelta=0.0
Armijo: th(0.087371958051152)=1.3392379381061466E-9; dx=-1.511100470972335E-8 evalInputDelta=0.0
Armijo: th(0.029123986017050667)=1.3392379381061466E-9; dx=-1.3821708343356615E-8 evalInputDelta=0.0
Armijo: th(0.007280996504262667)=1.3392379381061466E-9; dx=-1.3384777578567728E-8 evalInputDelta=0.0
Armijo: th(0.0014561993008525335)=1.3392379381061466E-9; dx=-1.327213886188587E-8 evalInputDelta=0.0
Armijo: th(2.4269988347542224E-4)=1.3392379381061466E-9; dx=-1.3248871229019586E-8 evalInputDelta=0.0
Armijo: th(3.467141192506032E-5)=1.3392379381061466E-9; dx=-1.3244889317926202E-8 evalInputDelta=0.0
Armijo: th(4.33392649063254E-6)=1.3392379381061466E-9; dx=-1.3244308789204713E-8 evalInputDelta=0.0
Armijo: th(4.8154738784806E-7)=1.3392379381061466E-9; dx=-1.3244235074307256E-8 evalInputDelta=0.0
Armijo: th(4.8154738784806E-8)=1.3392379381061466E-9; dx=-1.324422678142436E-8 evalInputDelta=0.0
Armijo: th(4.377703525891455E-9)=1.3392379381061466E-9; dx=-1.3244225943759478E-8 evalInputDelta=0.0
Armijo: th(3.6480862715762125E-10)=1.3392379381061466E-9; dx=-1.3244225866969545E-8 evalInputDelta=0.0
Armijo: th(2.806220208904779E-11)=1.3392379381061466E-9; dx=-1.3244225860532272E-8 evalInputDelta=0.0
WOLFE (weak): th(2.0044430063605565E-12)=1.3392379381061466E-9; dx=-1.3244225860029563E-8 evalInputDelta=0.0
Armijo: th(1.5033322547704172E-11)=1.3392379381061466E-9; dx=-1.324422586028182E-8 evalInputDelta=0.0
Armijo: th(8.518882777032364E-12)=1.3392379381061466E-9; dx=-1.3244225860156598E-8 evalInputDelta=0.0
WOLFE (weak): th(5.2616628916964606E-12)=1.3392379381061466E-9; dx=-1.3244225860093079E-8 evalInputDelta=0.0
WOLFE (weak): th(6.890272834364412E-12)=1.3392379381061466E-9; dx=-1.3244225860120304E-8 evalInputDelta=0.0
WOLFE (weak): th(7.704577805698388E-12)=1.3392379381061466E-9; dx=-1.3244225860138453E-8 evalInputDelta=0.0
Armijo: th(8.111730291365377E-12)=1.3392379381061466E-9; dx=-1.3244225860151153E-8 evalInputDelta=0.0
Armijo: th(7.908154048531883E-12)=1.3392379381061466E-9; dx=-1.3244225860146614E-8 evalInputDelta=0.0
WOLFE (weak): th(7.806365927115135E-12)=1.3392379381061466E-9; dx=-1.3244225860146615E-8 evalInputDelta=0.0
Armijo: th(7.857259987823508E-12)=1.3392379381061466E-9; dx=-1.3244225860146615E-8 evalInputDelta=0.0
mu ~= nu (7.806365927115135E-12): th(0.0)=1.3392379381061466E-9
Fitness changed from 1.3392379381061466E-9 to 1.3392379381061466E-9
Static Iteration Total: 0.0537; Orientation: 0.0002; Line Search: 0.0517
Iteration 7 failed. Error: 1.3392379381061466E-9
Previous Error: 0.0 -> 1.3392379381061466E-9
Optimization terminated 7
Final threshold in iteration 7: 1.3392379381061466E-9 (> 0.0) after 0.125s (< 30.000s)

Returns

    1.3392379381061466E-9

Training Converged

Conjugate Gradient Descent

First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.

TrainingTester.java:452 executed in 0.88 seconds (0.188 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
      iterativeTrainer.setOrientation(new GradientDescent());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 3725806733503
Reset training subject: 3725808499418
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=0.28015023395400235}, derivative=-0.36243331365923415}
New Minimum: 0.28015023395400235 > 0.280150233917759
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=0.280150233917759}, derivative=-0.3624333138737072}, evalInputDelta = -3.62433416611907E-11
New Minimum: 0.280150233917759 > 0.28015023370029896
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=0.28015023370029896}, derivative=-0.3624333151605456}, evalInputDelta = -2.537033916283349E-10
New Minimum: 0.28015023370029896 > 0.28015023217807905
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=0.28015023217807905}, derivative=-0.362433324168415}, evalInputDelta = -1.7759232973091343E-9
New Minimum: 0.28015023217807905 > 0.2801502215225383
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=0.2801502215225383}, derivative=-0.36243338722351165}, evalInputDelta = -1.2431464024853511E-8
New Minimum: 0.2801502215225383 > 0.28015014693370177
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=0.28015014693370177}, derivative=-0.3624338286097243}, evalInputDelta = -8.702030057650134E-8
New Minimum: 0.28015014693370177 > 0.2801496248093028
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=0.2801496248093028}, derivative=-0.36243691833950065}, evalInputDelta = -6.091446995704075E-7
New Minimum: 0.2801496248093028 > 0.28014596981387513
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=0.28014596981387513}, derivative=-0.3624585477360621}, evalInputDelta = -4.264140127219207E-6
New Minimum: 0.28014596981387513 > 0.2801203787370657
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=0.2801203787370657}, derivative=-0.36261001665561615}, evalInputDelta = -2.985521693665838E-5
New Minimum: 0.2801203787370657 > 0.2799409412734001
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=0.2799409412734001}, derivative=-0.3636734018521776}, evalInputDelta = -2.092926806022688E-4
New Minimum: 0.2799409412734001 > 0.27866997584855147
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=0.27866997584855147}, derivative=-0.37127218357676706}, evalInputDelta = -0.0014802581054508823
New Minimum: 0.27866997584855147 > 0.26896378893943923
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=0.26896378893943923}, derivative=-0.43327056531664504}, evalInputDelta = -0.011186445014563118
New Minimum: 0.26896378893943923 > 0.04495914041830033
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=0.04495914041830033}, derivative=-4.610521883037825}, evalInputDelta = -0.23519109353570203
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.17022326486268208
F(0.10647144000769232) = LineSearchPoint{point=PointSample{avg=0.2219937662299442}, derivative=-0.8469807860496746}, evalInputDelta = -0.058156467724058136
F(0.7453000800538463) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.17022326486268208
F(0.05733077538875741) = LineSearchPoint{point=PointSample{avg=0.2549633378661528}, derivative=-0.5358268978517876}, evalInputDelta = -0.025186896087849575
F(0.40131542772130185) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.17022326486268208
F(0.03087041751702322) = LineSearchPoint{point=PointSample{avg=0.26781718895417955}, derivative=-0.44107175133659693}, evalInputDelta = -0.012333044999822795
F(0.21609292261916255) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.17022326486268208
F(0.01662253250916635) = LineSearchPoint{point=PointSample{avg=0.2738112983554405}, derivative=-0.4014167629806087}, evalInputDelta = -0.006338935598561857
F(0.11635772756416446) = LineSearchPoint{point=PointSample{avg=0.21313098127516347}, derivative=-0.9493271818935728}, evalInputDelta = -0.06701925267883888
F(0.8145040929491512) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.17022326486268208
F(0.06265416099608855) = LineSearchPoint{point=PointSample{avg=0.2520493636219704}, derivative=-0.5592456231913424}, evalInputDelta = -0.028100870332031957
F(0.4385791269726198) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.17022326486268208
F(0.033736855920970754) = LineSearchPoint{point=PointSample{avg=0.26654033035648006}, derivative=-0.4498813503802505}, evalInputDelta = -0.013609903597522288
F(0.23615799144679528) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.17022326486268208
F(0.018165999342061177) = LineSearchPoint{point=PointSample{avg=0.27318865396610575}, derivative=-0.40540750067356734}, evalInputDelta = -0.006961579987896593
F(0.12716199539442824) = LineSearchPoint{point=PointSample{avg=0.2021533229589048}, derivative=-1.0882218333544111}, evalInputDelta = -0.07799691099509753
F(0.8901339677609976) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.17022326486268208
F(0.0684718436739229) = LineSearchPoint{point=PointSample{avg=0.24871639672311244}, derivative=-0.5869478237797855}, evalInputDelta = -0.03143383723088991
F(0.47930290571746026) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.17022326486268208
F(0.03686945428595848) = LineSearchPoint{point=PointSample{avg=0.2651154946035922}, derivative=-0.4598652678998432}, evalInputDelta = -0.015034739350410165
F(0.25808618000170935) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.17022326486268208
F(0.019852783077054566) = LineSearchPoint{point=PointSample{avg=0.27250108548563245}, derivative=-0.4098485712905961}, evalInputDelta = -0.007649148468369893
F(0.13896948153938196) = LineSearchPoint{point=PointSample{avg=0.1881959539659557}, derivative=-1.2853395716620626}, evalInputDelta = -0.09195427998804664
F(0.9727863707756738) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.17022326486268208
F(0.07482972082889798) = LineSearchPoint{point=PointSample{avg=0.2448810581709045}, derivative=-0.6200621019982377}, evalInputDelta = -0.03526917578309785
F(0.5238080458022859) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.17022326486268208
F(0.04029292660017584) = LineSearchPoint{point=PointSample{avg=0.26352185037532394}, derivative=-0.47122561858821577}, evalInputDelta = -0.016628383578678407
F(0.28205048620123085) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.17022326486268208
F(0.02169619124624853) = LineSearchPoint{point=PointSample{avg=0.27174101954076385}, derivative=-0.41479990686139356}, evalInputDelta = -0.008409214413238497
F(0.1518733387237397) = LineSearchPoint{point=PointSample{avg=0.16980641530683616}, derivative=-1.5825175070911228}, evalInputDelta = -0.11034381864716619
F(1.063113371066178) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.17022326486268208
F(0.08177795162047523) = LineSearchPoint{point=PointSample{avg=0.24043601813087173}, derivative=-0.660137660784866}, evalInputDelta = -0.039714215823130616
F(0.5724456613433266) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.17022326486268208
F(0.044034281641794355) = LineSearchPoint{point=PointSample{avg=0.2617347280302652}, derivative=-0.4842110568644082}, evalInputDelta = -0.01841550592373714
F(0.3082399714925605) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.17022326486268208
F(0.02371076703788927) = LineSearchPoint{point=PointSample{avg=0.2708998232792342}, derivative=-0.4203314510043829}, evalInputDelta = -0.009250410674768172
F(0.16597536926522488) = LineSearchPoint{point=PointSample{avg=0.1443117370408068}, derivative=-2.070074967391466}, evalI

...skipping 19042 bytes...


    Loops = 52
    Converged to right
    Fitness changed from 2.954073897008461E-8 to 1.3392379381061466E-9
    Iteration 3 complete. Error: 1.3392379381061466E-9 Total: 0.1745; Orientation: 0.0007; Line Search: 0.1713
    F(0.0) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-3.321431829287613E-4}
    F(0.41668369689825374) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(0.03205259206909644) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(0.002465584005315111) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(1.896603081011624E-4) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-0.005603524839949892}, evalInputDelta = 0.0
    F(0.0013276221567081368) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(1.021247812852413E-4) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-8.291812507939552E-4}, evalInputDelta = 0.0
    F(7.148734689966891E-4) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(5.4990266845899165E-5) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-5.071659596425629E-4}, evalInputDelta = 0.0
    F(3.8493186792129417E-4) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(2.9610143686253396E-5) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-4.1099534593973345E-4}, evalInputDelta = 0.0
    F(2.0727100580377378E-4) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-0.016741801163528244}, evalInputDelta = 0.0
    F(0.0014508970406264163) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(1.1160746466357048E-4) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-9.366324754373633E-4}, evalInputDelta = 0.0
    F(7.812522526449933E-4) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(6.009632712653795E-5) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-5.310987744592171E-4}, evalInputDelta = 0.0
    F(4.206742898857657E-4) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(3.235956076044351E-5) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-4.198873978648287E-4}, evalInputDelta = 0.0
    F(2.2651692532310458E-4) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(1.7424378871008044E-5) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-3.751127185770416E-4}, evalInputDelta = 0.0
    F(1.2197065209705631E-4) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-0.0010833961993753393}, evalInputDelta = 0.0
    F(8.537945646793941E-4) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(6.5676504975338E-5) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-5.594840495284604E-4}, evalInputDelta = 0.0
    F(4.5973553482736606E-4) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(3.536427190979739E-5) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-4.2997774367998213E-4}, evalInputDelta = 0.0
    F(2.475499033685817E-4) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(1.9042300259121672E-5) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-3.7957050575857047E-4}, evalInputDelta = 0.0
    F(1.332961018138517E-4) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-0.0012933211573137725}, evalInputDelta = 0.0
    F(9.33072712696962E-4) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(7.177482405361246E-5) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-5.935152136136161E-4}, evalInputDelta = 0.0
    F(5.024237683752872E-4) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(3.86479821827144E-5) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-4.4147553223510715E-4}, evalInputDelta = 0.0
    F(2.7053587527900083E-4) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(2.0810451944538524E-5) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-3.8454408093143295E-4}, evalInputDelta = 0.0
    F(1.4567316361176967E-4) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-0.0016131117926203493}, evalInputDelta = 0.0
    F(0.0010197121452823876) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(7.84393957909529E-5) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-6.348369069751695E-4}, evalInputDelta = 0.0
    F(5.490757705366703E-4) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(4.223659773359002E-5) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-4.5463874251076706E-4}, evalInputDelta = 0.0
    F(2.9565618413513015E-4) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(2.2742783395010012E-5) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-3.9010489804651476E-4}, evalInputDelta = 0.0
    F(1.5919948376507008E-4) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-0.002146604712985075}, evalInputDelta = 0.0
    F(0.0011143963863554905) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(8.572279895042234E-5) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-6.857754214363253E-4}, evalInputDelta = 0.0
    F(6.000595926529565E-4) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(4.615843020407357E-5) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-4.697891029008032E-4}, evalInputDelta = 0.0
    F(3.23109011428515E-4) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(2.4854539340655002E-5) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-3.96336980965022E-4}, evalInputDelta = 0.0
    F(1.7398177538458502E-4) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-0.00317276090448357}, evalInputDelta = 0.0
    F(0.0012178724276920951) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(9.368249443785347E-5) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-7.497235041198667E-4}, evalInputDelta = 0.0
    F(6.557774610649743E-4) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    F(5.04444200819211E-5) = LineSearchPoint{point=PointSample{avg=1.3392379381061466E-9}, derivative=-4.873327718630275E-4}, evalInputDelta = 0.0
    F(3.531109405734477E-4) = LineSearchPoint{point=PointSample{avg=0.4503734988166844}, derivative=0.0}, evalInputDelta = 0.4503734974774465
    Loops = 52
    Converged to right
    Fitness changed from 1.3392379381061466E-9 to 1.3392379381061466E-9
    Static Iteration Total: 0.3472; Orientation: 0.0004; Line Search: 0.3441
    Iteration 4 failed. Error: 1.3392379381061466E-9
    Previous Error: 0.0 -> 1.3392379381061466E-9
    Optimization terminated 4
    Final threshold in iteration 4: 1.3392379381061466E-9 (> 0.0) after 0.876s (< 30.000s)

Returns

    1.3392379381061466E-9

Training Converged

Limited-Memory BFGS

Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.

TrainingTester.java:509 executed in 0.16 seconds (0.000 gc):

    IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
    try {
      iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
      iterativeTrainer.setOrientation(new LBFGS());
      iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
      iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
      iterativeTrainer.setIterationsPerSample(100);
      iterativeTrainer.setMaxIterations(250);
      iterativeTrainer.setTerminateThreshold(0);
      return iterativeTrainer.run();
    } finally {
      iterativeTrainer.freeRef();
    }
Logging
Reset training subject: 3726690812757
Reset training subject: 3726692733595
Adding measurement 37c0fe47 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 0.28015023395400235 < 0.28015023395400235. Total: 1
th(0)=0.28015023395400235;dx=-0.36243331365923415
Non-optimal measurement 0.4503734988166844 < 0.28015023395400235. Total: 1
Armijo: th(2.154434690031884)=0.4503734988166844; dx=0.0 evalInputDelta=-0.17022326486268208
Non-optimal measurement 0.4503734988166844 < 0.28015023395400235. Total: 1
Armijo: th(1.077217345015942)=0.4503734988166844; dx=0.0 evalInputDelta=-0.17022326486268208
Non-optimal measurement 0.4503734988166844 < 0.28015023395400235. Total: 1
Armijo: th(0.3590724483386473)=0.4503734988166844; dx=0.0 evalInputDelta=-0.17022326486268208
Adding measurement 5d18608d to history. Total: 1
New Minimum: 0.28015023395400235 > 0.2349583540168329
WOLFE (weak): th(0.08976811208466183)=0.2349583540168329; dx=-0.7121026059724442 evalInputDelta=0.045191879937169444
Non-optimal measurement 0.4503734988166844 < 0.2349583540168329. Total: 2
Armijo: th(0.22442028021165458)=0.4503734988166844; dx=0.0 evalInputDelta=-0.17022326486268208
Adding measurement 23c904db to history. Total: 2
New Minimum: 0.2349583540168329 > 0.16114929598092997
WOLFE (weak): th(0.1570941961481582)=0.16114929598092997; dx=-1.7379704823916264 evalInputDelta=0.11900093797307237
Adding measurement 3f3cf18e to history. Total: 3
New Minimum: 0.16114929598092997 > 0.07446957176321886
WOLFE (weak): th(0.1907572381799064)=0.07446957176321886; dx=-3.84134929079155 evalInputDelta=0.2056806621907835
Adding measurement 30e9d814 to history. Total: 4
New Minimum: 0.07446957176321886 > 0.002039857293237384
WOLFE (weak): th(0.2075887591957805)=0.002039857293237384; dx=-2.2408747063251764 evalInputDelta=0.278110376660765
Non-optimal measurement 0.4503734988166844 < 0.002039857293237384. Total: 5
Armijo: th(0.21600451970371753)=0.4503734988166844; dx=0.0 evalInputDelta=-0.17022326486268208
Non-optimal measurement 0.4503734988166844 < 0.002039857293237384. Total: 5
Armijo: th(0.21179663944974902)=0.4503734988166844; dx=0.0 evalInputDelta=-0.17022326486268208
Adding measurement 61280e89 to history. Total: 5
New Minimum: 0.002039857293237384 > 1.502688973835706E-6
END: th(0.20969269932276474)=1.502688973835706E-6; dx=-0.018856070265211758 evalInputDelta=0.28014873126502854
Fitness changed from 0.28015023395400235 to 1.502688973835706E-6
Iteration 1 complete. Error: 1.502688973835706E-6 Total: 0.0476; Orientation: 0.0010; Line Search: 0.0410
Non-optimal measurement 1.502688973835706E-6 < 1.502688973835706E-6. Total: 6
Accepted: LBFGS Orientation magnitude: 2.888e-03, gradient 3.259e-02, dot 0.971; [e61ac1c1-5af1-4ace-bd81-7040a0529dbc = 1.000/1.000e+00, 26b0c225-e17b-4c9b-b98d-d45e79e854d4 = 1.000/1.000e+00, 6f4efd03-98a6-40bf-9ffa-e3dd9028b2f5 = 1.000/1.000e+00, 3783fab8-fba4-4752-84dd-558491a370c8 = 1.000/1.000e+00, 271f956b-a4bf-4b72-b8f4-a2f61ddc7eeb = 1.000/1.000e+00, 0b13074d-f0fd-40ac-a45a-152981e128c8 = 1.000/1.000e+00]
Constructing line search parameters: LBFGS
Non-optimal measurement 1.502688973835706E-6 < 1.502688973835706E-6. Total: 6
th(0)=1.502688973835706E-6;dx=-9.143321933286187E-5
Non-optimal measurement 0.4503734988166844 < 1.502688973835706E-6. Total: 6
Armijo: th(2.154434690031884)=0.4503734988166844; dx=0.0 evalInputDelta=-0.4503719961277106
Non-optimal measurement 0.4503734988166844 < 1.502688973835706E-6. Total: 6
Armijo: th(1.077217345015942)=0.4503734988166844; dx=0.0 evalInputDelta=-0.4503719961277106
Non-optimal measurement 0.4503734988166844 < 1.502688973835706E-6. Total: 6
Armijo: th(0.3590724483386473)=0.4503734988166844; dx=0.0 evalInputDelta=-0.4503719961277106
Adding measurement 1b118e9d to history. Total: 6
New Minimum: 1.502688973835706E-6 > 1.3392379381061466E-9
END: th(0.08976811208466183)=1.3392379381061466E-9; dx=-1.293248708682948E-6 evalInputDelta=1.5013497358975998E-6
Fitness changed from 1.502688973835706E-6 to 1.3392379381061466E-9
Iteration 2 complete. Error: 1.3392379381061466E-9 Total: 0.0256; Orientation: 0.0130; Line Search: 0.0111
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
Accepted: LBFGS Orientation magnitude: 4.085e-05, gradient 4.610e-04, dot 0.971; [26b0c225-e17b-4c9b-b98d-d45e79e854d4 = 1.000/1.000e+00, 6f4efd03-98a6-40bf-9ffa-e3dd9028b2f5 = 1.000/1.000e+00, 3783fab8-fba4-4752-84dd-558491a370c8 = 1.000/1.000e+00, e61ac1c1-5af1-4ace-bd81-7040a0529dbc = 1.000/1.000e+00, 0b13074d-f0fd-40ac-a45a-152981e128c8 = 1.000/1.000e+00, 271f956b-a4bf-4b72-b8f4-a2f61ddc7eeb = 1.000/1.000e+00]
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
th(0)=1.3392379381061466E-9;dx=-1.829127713122578E-8
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
Armijo: th(0.1933995347338658)=1.3392379381061466E-9; dx=-2.38427846652369E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
Armijo: th(0.0966997673669329)=1.3392379381061466E-9; dx=-2.0761615096254208E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
Armijo: th(0.032233255788977636)=1.3392379381061466E-9; dx=-1.9057754569149745E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
Armijo: th(0.008058313947244409)=1.3392379381061466E-9; dx=-1.84780196226539E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
Armijo: th(0.0016116627894488817)=1.3392379381061466E-9; dx=-1.832837331287339E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
Armijo: th(2.6861046490814694E-4)=1.3392379381061466E-9; dx=-1.8297451133862056E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
Armijo: th(3.8372923558306706E-5)=1.3392379381061466E-9; dx=-1.829215891902612E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
Armijo: th(4.796615444788338E-6)=1.3392379381061466E-9; dx=-1.8291387350833105E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
Armijo: th(5.329572716431487E-7)=1.3392379381061466E-9; dx=-1.8291289377806087E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
Armijo: th(5.329572716431487E-8)=1.3392379381061466E-9; dx=-1.8291278355880665E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
Armijo: th(4.845066105846806E-9)=1.3392379381061466E-9; dx=-1.8291277242561193E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
Armijo: th(4.037555088205672E-10)=1.3392379381061466E-9; dx=-1.8291277140506097E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
Armijo: th(3.1058116063120555E-11)=1.3392379381061466E-9; dx=-1.8291277131946455E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
WOLFE (weak): th(2.2184368616514684E-12)=1.3392379381061466E-9; dx=-1.8291277131282673E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
Armijo: th(1.663827646238601E-11)=1.3392379381061466E-9; dx=-1.82912771316114E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
Armijo: th(9.42835666201874E-12)=1.3392379381061466E-9; dx=-1.8291277131440714E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
Armijo: th(5.823396761835104E-12)=1.3392379381061466E-9; dx=-1.8291277131361695E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
WOLFE (weak): th(4.0209168117432864E-12)=1.3392379381061466E-9; dx=-1.8291277131314287E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
WOLFE (weak): th(4.922156786789196E-12)=1.3392379381061466E-9; dx=-1.8291277131345892E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
WOLFE (weak): th(5.37277677431215E-12)=1.3392379381061466E-9; dx=-1.8291277131345892E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
WOLFE (weak): th(5.5980867680736276E-12)=1.3392379381061466E-9; dx=-1.8291277131361698E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
Armijo: th(5.710741764954366E-12)=1.3392379381061466E-9; dx=-1.8291277131361695E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
Armijo: th(5.654414266513997E-12)=1.3392379381061466E-9; dx=-1.8291277131361695E-8 evalInputDelta=0.0
Non-optimal measurement 1.3392379381061466E-9 < 1.3392379381061466E-9. Total: 7
mu ~= nu (5.5980867680736276E-12): th(0.0)=1.3392379381061466E-9
Fitness changed from 1.3392379381061466E-9 to 1.3392379381061466E-9
Static Iteration Total: 0.0843; Orientation: 0.0152; Line Search: 0.0675
Iteration 3 failed. Error: 1.3392379381061466E-9
Previous Error: 0.0 -> 1.3392379381061466E-9
Optimization terminated 3
Final threshold in iteration 3: 1.3392379381061466E-9 (> 0.0) after 0.158s (< 30.000s)

Returns

    1.3392379381061466E-9

Training Converged

TrainingTester.java:432 executed in 0.01 seconds (0.000 gc):

    return TestUtil.compare(title + " vs Iteration", runs);
Logging
Plotting range=[1.0, -8.8731422564261], [6.0, -1.3471820007120736]; valueStats=DoubleSummaryStatistics{count=11, sum=0.044962, min=0.000000, average=0.004087, max=0.044959}
Plotting 6 points for GD
Plotting 3 points for CjGD
Plotting 2 points for LBFGS

Returns

Result

TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):

    return TestUtil.compareTime(title + " vs Time", runs);
Logging
Plotting range=[0.0, -8.8731422564261], [0.369, -1.3471820007120736]; valueStats=DoubleSummaryStatistics{count=11, sum=0.044962, min=0.000000, average=0.004087, max=0.044959}
Plotting 6 points for GD
Plotting 3 points for CjGD
Plotting 2 points for LBFGS

Returns

Result

Results

TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):

    return grid(inputLearning, modelLearning, completeLearning);

Returns

Result

TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):

    return new ComponentResult(null == inputLearning ? null : inputLearning.value,
        null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);

Returns

    {"input":{ "LBFGS": { "type": "NonConverged", "value": 1.3392379381061466E-9 }, "CjGD": { "type": "NonConverged", "value": 1.3392379381061466E-9 }, "GD": { "type": "NonConverged", "value": 1.3392379381061466E-9 } }, "model":{ "LBFGS": { "type": "NonConverged", "value": 1.0305768471459636E-9 }, "CjGD": { "type": "NonConverged", "value": 1.0305768471459636E-9 }, "GD": { "type": "NonConverged", "value": 1.0305768471459636E-9 } }, "complete":{ "LBFGS": { "type": "NonConverged", "value": 1.3392379381061466E-9 }, "CjGD": { "type": "NonConverged", "value": 1.3392379381061466E-9 }, "GD": { "type": "NonConverged", "value": 1.3392379381061466E-9 } }}

LayerTests.java:425 executed in 0.00 seconds (0.000 gc):

    throwException(exceptions.addRef());

Results

detailsresult
{"input":{ "LBFGS": { "type": "NonConverged", "value": 1.3392379381061466E-9 }, "CjGD": { "type": "NonConverged", "value": 1.3392379381061466E-9 }, "GD": { "type": "NonConverged", "value": 1.3392379381061466E-9 } }, "model":{ "LBFGS": { "type": "NonConverged", "value": 1.0305768471459636E-9 }, "CjGD": { "type": "NonConverged", "value": 1.0305768471459636E-9 }, "GD": { "type": "NonConverged", "value": 1.0305768471459636E-9 } }, "complete":{ "LBFGS": { "type": "NonConverged", "value": 1.3392379381061466E-9 }, "CjGD": { "type": "NonConverged", "value": 1.3392379381061466E-9 }, "GD": { "type": "NonConverged", "value": 1.3392379381061466E-9 } }}OK
  {
    "result": "OK",
    "performance": {
      "execution_time": "13.282",
      "gc_time": "0.421"
    },
    "created_on": 1586738302029,
    "file_name": "trainingTest",
    "report": {
      "simpleName": "Normal",
      "canonicalName": "com.simiacryptus.mindseye.layers.java.SignReducerLayerTest.Normal",
      "link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/test/java/com/simiacryptus/mindseye/layers/java/SignReducerLayerTest.java",
      "javaDoc": ""
    },
    "training_analysis": {
      "input": {
        "LBFGS": {
          "type": "NonConverged",
          "value": 1.3392379381061466E-9
        },
        "CjGD": {
          "type": "NonConverged",
          "value": 1.3392379381061466E-9
        },
        "GD": {
          "type": "NonConverged",
          "value": 1.3392379381061466E-9
        }
      },
      "model": {
        "LBFGS": {
          "type": "NonConverged",
          "value": 1.0305768471459636E-9
        },
        "CjGD": {
          "type": "NonConverged",
          "value": 1.0305768471459636E-9
        },
        "GD": {
          "type": "NonConverged",
          "value": 1.0305768471459636E-9
        }
      },
      "complete": {
        "LBFGS": {
          "type": "NonConverged",
          "value": 1.3392379381061466E-9
        },
        "CjGD": {
          "type": "NonConverged",
          "value": 1.3392379381061466E-9
        },
        "GD": {
          "type": "NonConverged",
          "value": 1.3392379381061466E-9
        }
      }
    },
    "archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/java/SignReducerLayer/Normal/trainingTest/202004133822",
    "id": "2dda0b84-326d-4120-9b61-9f41bcf003c1",
    "report_type": "Components",
    "display_name": "Comparative Training",
    "target": {
      "simpleName": "SignReducerLayer",
      "canonicalName": "com.simiacryptus.mindseye.layers.java.SignReducerLayer",
      "link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/main/java/com/simiacryptus/mindseye/layers/java/SignReducerLayer.java",
      "javaDoc": ""
    }
  }