Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase
Using Seed 6709949179819475968
In this apply, we use a network to learn this target input, given it's pre-evaluated output:
TrainingTester.java:332 executed in 0.03 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
try {
return x.prettyPrint();
} finally {
x.freeRef();
}
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ -1.892, 1.0, -1.876 ], [ -1.388, 0.284, -0.464 ], [ -0.96, -0.144, -0.54 ], [ 0.724, 1.108, 0.14 ], [ -1.656, -0.824, -1.364 ], [ -1.836, -0.08, -1.252 ], [ -0.616, 1.112, -0.152 ], [ -1.744, -0.464, -0.772 ], ... ],
[ [ -0.736, 1.608, -1.96 ], [ 1.364, 0.324, 1.796 ], [ 0.78, 1.892, 0.652 ], [ 1.376, 1.972, -0.468 ], [ -0.656, -0.748, 0.776 ], [ -1.524, 1.388, -0.432 ], [ 0.44, 0.832, -0.072 ], [ 0.196, -1.332, 0.716 ], ... ],
[ [ 0.8, 0.524, 1.376 ], [ -0.408, 0.648, -0.604 ], [ -0.012, 0.744, 0.6 ], [ -1.324, -0.892, -0.296 ], [ -0.876, 1.504, -1.344 ], [ -0.056, -1.612, -0.568 ], [ 0.876, -0.284, 1.1 ], [ 0.28, -0.68, -1.268 ], ... ],
[ [ 0.6, -1.4, -0.296 ], [ -0.32, -1.896, 0.3 ], [ 1.576, -0.16, -1.772 ], [ -0.708, 1.392, -1.344 ], [ -0.716, 1.832, -0.904 ], [ 0.404, -0.356, 1.392 ], [ -1.34, 0.036, 1.848 ], [ 0.768, -1.328, 1.004 ], ... ],
[ [ -0.468, 1.232, -1.476 ], [ 1.228, 1.268, -0.388 ], [ -1.448, 1.22, 0.144 ], [ 0.344, 0.74, 1.912 ], [ -0.78, 0.232, -0.324 ], [ 1.288, -1.712, -0.168 ], [ -1.128, 1.288, 0.72 ], [ 1.112, -1.372, -1.352 ], ... ],
[ [ -1.712, 1.912, 1.76 ], [ -0.568, -1.192, 0.472 ], [ 1.932, 0.024, -1.028 ], [ -0.028, 0.004, -1.936 ], [ -0.196, 1.384, 0.404 ], [ -1.66, -1.928, 1.684 ], [ 1.604, -0.16, 0.232 ], [ 0.256, -0.732, 0.912 ], ... ],
[ [ 0.432, -0.988, -0.388 ], [ -0.548, -0.772, -1.348 ], [ 1.588, -1.504, 1.52 ], [ 0.168, -1.936, 0.308 ], [ -1.56, -1.072, -1.732 ], [ -0.496, 1.312, -0.084 ], [ 1.596, 1.62, 0.128 ], [ 1.368, -0.292, 0.844 ], ... ],
[ [ -0.808, 0.288, -1.88 ], [ 1.308, 1.768, -1.752 ], [ -0.66, -0.208, -0.852 ], [ 0.044, 0.496, -0.092 ], [ 1.716, -0.276, -0.424 ], [ 0.08, -0.8, -1.684 ], [ -1.996, 1.544, 0.556 ], [ -0.108, 0.008, -1.5 ], ... ],
...
]
[
[ [ 0.44, -0.804, -0.832 ] ]
]
[
[ [ 1.484, 0.296, 1.804 ], [ -0.008, 1.62, -1.212 ], [ 0.58, 0.628, 1.328 ], [ -0.468, 0.252, 0.396 ], [ 1.932, 1.808, -1.824 ], [ -1.532, 0.868, 1.56 ], [ 1.656, 1.672, 1.344 ], [ 1.512, -1.32, -1.932 ], ... ],
[ [ -0.984, 1.548, 1.048 ], [ -0.144, -0.084, -0.184 ], [ 0.468, 1.476, -0.268 ], [ 1.036, 1.376, -1.844 ], [ -0.884, 0.504, 0.54 ], [ -1.416, -0.312, 0.256 ], [ -1.94, 0.996, -1.536 ], [ 1.34, 1.356, -0.76 ], ... ],
[ [ 1.824, 0.492, 1.688 ], [ -1.644, 1.624, -0.244 ], [ -1.44, 1.072, 0.796 ], [ 1.904, -0.764, -0.052 ], [ -0.772, -1.764, 1.736 ], [ -1.308, 1.676, 0.564 ], [ 0.54, 0.492, -1.764 ], [ -1.824, 1.576, -1.336 ], ... ],
[ [ 1.928, 0.08, 0.048 ], [ 0.796, -0.968, -1.54 ], [ 1.516, -0.592, -0.052 ], [ -1.804, -1.22, 1.892 ], [ 1.884, -1.992, -1.86 ], [ -0.724, -1.76, -1.888 ], [ 1.22, -0.856, -0.14 ], [ 1.152, -1.16, -0.284 ], ... ],
[ [ -0.268, -1.068, 1.8 ], [ 1.512, -0.36, -0.156 ], [ 0.548, 1.96, 1.08 ], [ 0.424, -1.712, -1.232 ], [ -0.072, 0.712, -1.448 ], [ 0.94, 0.488, 0.912 ], [ -1.624, 0.4, 0.468 ], [ 1.624, -1.916, 0.796 ], ... ],
[ [ -0.124, 1.204, 1.216 ], [ -1.736, -0.344, 0.02 ], [ 1.548, -0.476, -0.396 ], [ 1.22, -0.324, 1.336 ], [ 1.708, -0.204, -0.74 ], [ -1.484, 0.968, 0.552 ], [ 0.988, -0.644, 1.504 ], [ -1.38, 0.704, -1.168 ], ... ],
[ [ 1.468, 0.78, 1.028 ], [ -0.372, -1.716, 0.42 ], [ -1.836, 1.772, 0.104 ], [ 1.096, 0.3, -1.136 ], [ -0.108, -1.112, -1.956 ], [ 1.336, 0.372, 0.576 ], [ 1.352, -0.576, 1.924 ], [ -1.808, 0.28, -1.808 ], ... ],
[ [ -0.24, -1.708, -1.696 ], [ -1.912, -0.432, 0.636 ], [ -0.516, -0.376, -0.512 ], [ 0.836, -1.732, 0.848 ], [ -0.012, 0.764, -1.272 ], [ 1.424, -1.736, 0.416 ], [ -0.384, 1.36, -0.824 ], [ -0.228, 0.044, 0.88 ], ... ],
...
]
[
[ [ -0.832, 0.44, -0.804 ] ]
]
[
[ [ 1.292, -1.88, -0.328 ], [ -0.144, 0.604, -0.984 ], [ 0.512, 0.62, 1.984 ], [ 1.092, 0.308, 1.32 ], [ -0.74, 1.952, 1.532 ], [ 0.08, 1.016, -0.168 ], [ 1.324, 1.504, 1.288 ], [ 0.708, -1.308, 1.76 ], ... ],
[ [ 1.292, -0.596, 1.92 ], [ 1.788, 1.288, 0.764 ], [ -1.444, 0.676, -1.564 ], [ -0.348, 1.152, 1.676 ], [ 1.936, -0.484, -1.612 ], [ 0.564, -1.036, 1.508 ], [ -0.332, -1.0, 1.572 ], [ -0.028, -0.848, -1.856 ], ... ],
[ [ -1.2, 0.692, 1.604 ], [ -1.528, -0.376, 0.98 ], [ 0.58, 1.156, -0.308 ], [ -0.144, 0.604, -0.62 ], [ 0.132, 1.036, -1.96 ], [ -0.72, 0.404, -1.208 ], [ -1.36, 0.152, -0.964 ], [ 0.7, 0.296, 0.196 ], ... ],
[ [ 0.28, 0.016, 1.816 ], [ 1.136, 1.864, 1.76 ], [ -0.056, -1.456, 0.972 ], [ 0.54, 0.088, 0.46 ], [ -0.192, 0.304, 0.756 ], [ 0.984, -0.344, -1.42 ], [ 1.26, -1.116, 0.232 ], [ -1.992, 0.152, -0.62 ], ... ],
[ [ -1.788, -0.12, -1.028 ], [ -0.664, 1.16, 1.228 ], [ -1.212, -1.84, -0.456 ], [ -1.588, 0.708, 0.828 ], [ -1.744, 0.94, 0.9 ], [ 0.964, 1.66, 1.724 ], [ 0.928, -0.616, 1.208 ], [ -0.168, -0.28, 1.64 ], ... ],
[ [ 0.836, 1.468, 0.812 ], [ -0.524, -1.224, -0.052 ], [ -1.288, -1.168, 1.616 ], [ -1.008, -0.168, -0.304 ], [ -0.864, 1.604, 0.544 ], [ -0.468, -1.388, 1.504 ], [ -1.008, -1.656, -1.532 ], [ -0.852, 0.488, 1.872 ], ... ],
[ [ -1.348, 1.308, -1.424 ], [ 1.064, 1.28, 1.032 ], [ 0.38, 1.464, 0.36 ], [ -1.96, -0.444, -0.308 ], [ 0.924, 0.04, 0.86 ], [ -1.404, 0.34, 1.948 ], [ 0.796, -0.132, -0.768 ], [ 1.04, -0.124, -1.94 ], ... ],
[ [ 1.28, 0.624, -0.74 ], [ -1.92, 1.132, -1.108 ], [ 1.888, -0.792, -0.808 ], [ 0.768, -1.144, -0.864 ], [ -1.152, 0.788, 1.704 ], [ 0.992, -1.628, 0.66 ], [ -1.328, -0.308, -1.668 ], [ 1.652, 0.768, 1.632 ], ... ],
...
]
[
[ [ -0.832, 0.44, -0.804 ] ]
]
[
[ [ 1.92, -0.784, 1.24 ], [ -0.944, 0.428, 1.616 ], [ -1.564, 1.924, -1.4 ], [ 0.132, 1.168, -1.628 ], [ 1.684, -1.892, -0.524 ], [ -1.368, -1.632, 1.94 ], [ 1.804, -0.268, 1.632 ], [ 0.6, -1.584, -0.184 ], ... ],
[ [ -1.824, 1.352, 0.56 ], [ 0.828, -0.968, -0.976 ], [ -0.072, 1.048, 1.92 ], [ -1.924, 1.1, 0.976 ], [ 1.444, -1.852, -1.424 ], [ 0.916, -1.16, -0.608 ], [ -1.472, 0.124, 1.428 ], [ 0.86, 1.848, -0.864 ], ... ],
[ [ 1.328, 0.656, 0.752 ], [ -0.384, 0.128, -0.46 ], [ 0.768, -1.096, -1.86 ], [ -0.712, 0.36, -1.968 ], [ 0.62, -1.48, -1.292 ], [ -1.924, 1.796, -1.596 ], [ 0.044, 1.696, 1.54 ], [ -0.74, 1.744, -1.22 ], ... ],
[ [ -1.304, -0.488, 0.084 ], [ 0.976, -1.68, -0.852 ], [ -0.868, 1.396, -0.056 ], [ 0.816, -0.348, 0.044 ], [ 0.9, 1.108, 1.112 ], [ -0.488, 0.228, 0.46 ], [ 1.492, -1.012, -0.944 ], [ 0.156, -1.288, -0.316 ], ... ],
[ [ 1.096, -1.1, 0.668 ], [ 1.716, 1.2, 1.372 ], [ -0.42, 0.624, -1.668 ], [ -1.628, -0.068, 0.796 ], [ -1.872, -1.468, -0.42 ], [ 1.528, 0.584, -0.348 ], [ 0.26, -1.648, -1.656 ], [ 1.128, 1.328, 1.46 ], ... ],
[ [ -1.776, 1.524, 1.1 ], [ -1.556, -0.524, 1.808 ], [ 1.996, 0.98, -1.532 ], [ -0.004, 1.744, -1.036 ], [ 0.836, -1.964, -0.472 ], [ -1.764, -0.576, 0.152 ], [ 0.824, 1.16, -0.24 ], [ -0.748, 0.848, -0.636 ], ... ],
[ [ 1.28, -1.68, 1.764 ], [ -0.5, -0.412, -0.812 ], [ 0.948, -0.176, -0.58 ], [ 0.568, 0.54, -0.14 ], [ 0.832, -0.46, 1.068 ], [ -0.564, 0.048, -1.788 ], [ -0.372, -0.072, 1.528 ], [ 0.48, -1.744, 0.252 ], ... ],
[ [ -1.484, 1.228, -0.244 ], [ -1.296, 0.76, -1.256 ], [ -0.596, -1.0, -0.852 ], [ -1.844, -0.516, 0.892 ], [ 0.364, -0.804, -1.756 ], [ 0.612, 0.292, -1.892 ], [ 1.056, -0.26, 0.46 ], [ -1.148, 1.976, -1.64 ], ... ],
...
]
[
[ [ 0.44, -0.804, -0.832 ] ]
]
[
[ [ -0.972, -1.164, 1.456 ], [ 0.944, -1.94, -1.124 ], [ 1.784, -0.82, 1.888 ], [ 0.48, 0.092, -1.28 ], [ 1.928, -1.06, 1.048 ], [ -1.124, -1.496, -1.796 ], [ -1.416, -0.64, -1.908 ], [ -0.604, 1.196, -1.448 ], ... ],
[ [ -1.404, -1.352, -0.464 ], [ 0.084, -1.088, 0.528 ], [ 1.86, -1.856, -1.908 ], [ 1.004, 0.392, -0.704 ], [ 1.512, 0.74, -0.924 ], [ 1.188, 0.544, -1.052 ], [ -0.64, 0.364, -0.572 ], [ 0.572, -1.804, -1.368 ], ... ],
[ [ -0.608, -1.088, 1.848 ], [ -0.92, -1.908, -0.504 ], [ -1.484, -1.856, 0.848 ], [ -1.108, -0.068, -1.828 ], [ -0.916, -0.676, -1.496 ], [ 0.864, 1.404, -0.044 ], [ 0.256, -0.58, 0.972 ], [ -0.852, 1.644, -1.784 ], ... ],
[ [ 0.76, 0.852, 1.4 ], [ 1.332, -0.74, 0.212 ], [ 0.632, 0.944, 1.464 ], [ -0.78, -1.324, -1.804 ], [ -0.032, 0.424, -0.052 ], [ 1.76, 0.404, 1.608 ], [ -0.176, 1.676, -1.4 ], [ 1.896, -0.928, -1.86 ], ... ],
[ [ -0.544, -0.128, 1.604 ], [ -0.732, 0.108, -1.892 ], [ -0.456, 0.812, -0.176 ], [ 0.34, 1.112, 0.568 ], [ 0.26, 1.904, -1.22 ], [ 1.508, -1.884, -0.472 ], [ 1.064, 0.024, 1.552 ], [ -1.536, -0.652, 0.412 ], ... ],
[ [ 1.208, 0.496, -1.776 ], [ -1.124, -0.208, 1.392 ], [ 1.916, 0.74, 1.584 ], [ 0.448, 0.312, -1.804 ], [ -0.528, 1.208, -1.556 ], [ -0.412, 1.388, -0.536 ], [ -0.656, -0.628, -0.1 ], [ -0.22, 1.424, 1.352 ], ... ],
[ [ 0.608, -1.676, -0.748 ], [ 1.484, -0.96, 0.628 ], [ -0.112, 0.072, -0.1 ], [ -0.204, -1.208, -1.492 ], [ -1.284, -0.248, -1.516 ], [ -1.74, -0.404, 0.14 ], [ -1.008, -1.592, 0.808 ], [ 0.832, -0.304, -0.992 ], ... ],
[ [ -0.044, -0.268, 1.14 ], [ -1.852, -0.176, -0.536 ], [ -0.8, 0.432, 1.316 ], [ -1.652, -1.764, 0.08 ], [ 0.48, 0.576, 0.696 ], [ 1.564, -1.58, -0.18 ], [ 1.988, 0.52, -0.832 ], [ 1.348, -0.216, -0.184 ], ... ],
...
]
[
[ [ -0.804, 0.44, -0.832 ] ]
]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 30.38 seconds (2.611 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 5376403773005
Reset training subject: 5377540564238
Constructing line search parameters: GD
th(0)=3.298985639906153;dx=-0.16864947480671733
New Minimum: 3.298985639906153 > 2.9878280034704936
END: th(2.154434690031884)=2.9878280034704936; dx=-0.12020367707852045 evalInputDelta=0.3111576364356594
Fitness changed from 3.298985639906153 to 2.9878280034704936
Iteration 1 complete. Error: 2.9878280034704936 Total: 8.6211; Orientation: 0.6590; Line Search: 4.5452
th(0)=2.9878280034704936;dx=-0.08567433057099197
New Minimum: 2.9878280034704936 > 2.7132155352032257
END: th(4.641588833612779)=2.7132155352032257; dx=-0.032652596772015145 evalInputDelta=0.27461246826726793
Fitness changed from 2.9878280034704936 to 2.7132155352032257
Iteration 2 complete. Error: 2.7132155352032257 Total: 4.2332; Orientation: 0.6914; Line Search: 2.6808
th(0)=2.7132155352032257;dx=-0.01244489770259318
New Minimum: 2.7132155352032257 > 2.6717293049407016
WOLF (strong): th(10.000000000000002)=2.6717293049407016; dx=0.004147652549703677 evalInputDelta=0.04148623026252407
END: th(5.000000000000001)=2.6717317364913913; dx=-0.004148622844924993 evalInputDelta=0.041483798711834385
Fitness changed from 2.7132155352032257 to 2.6717293049407016
Iteration 3 complete. Error: 2.6717293049407016 Total: 5.4221; Orientation: 0.7005; Line Search: 3.9207
th(0)=2.6717293049407016;dx=-0.0013832131634255664
New Minimum: 2.6717293049407016 > 2.6675257917945885
WOLF (strong): th(10.772173450159421)=2.6675257917945885; dx=6.027739076559368E-4 evalInputDelta=0.004203513146113114
New Minimum: 2.6675257917945885 > 2.666953379661158
END: th(5.386086725079711)=2.666953379661158; dx=-3.9021957642869473E-4 evalInputDelta=0.004775925279543625
Fitness changed from 2.6717293049407016 to 2.666953379661158
Iteration 4 complete. Error: 2.666953379661158 Total: 6.7155; Orientation: 0.7664; Line Search: 4.9902
th(0)=2.666953379661158;dx=-1.1033979444170556E-4
New Minimum: 2.666953379661158 > 2.666659065495842
WOLF (strong): th(11.60397208403195)=2.666659065495842; dx=5.9613634518545325E-5 evalInputDelta=2.943141653157788E-4
New Minimum: 2.666659065495842 > 2.6665597003108523
END: th(5.801986042015975)=2.6665597003108523; dx=-2.536309275200205E-5 evalInputDelta=3.936793503056535E-4
Fitness changed from 2.666953379661158 to 2.6665597003108523
Iteration 5 complete. Error: 2.6665597003108523 Total: 5.3812; Orientation: 0.6852; Line Search: 3.9355
Final threshold in iteration 5: 2.6665597003108523 (> 0.0) after 30.374s (< 30.000s)
Returns
2.6665597003108523
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.01 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.6879933159493061, 0.6320009531196437, -1.0160022289819168 ], [ 0.40799534738510124, -1.171996227744386, -0.2840004670216127 ], [ -1.523998537577564, -0.1479999900471068, -1.171998363034136 ], [ 0.6320002394150634, 0.7200010049420322, -0.9639971400267797 ], [ 0.48799444567604633, 1.7479933352343184, -1.631999306201124 ], [ 0.5479938238083363, 1.347996299472676, 1.2359935526790289 ], [ -0.052001460358171624, -1.7119926830959524, -1.0039977929883945 ], [ -0.9760019889458447, 1.5799947033443102, 1.6479937288750843 ], ... ],
[ [ 0.5399966947662311, -1.8759909729581885, -0.004005068846361022 ], [ -1.1279935419033733, -1.4279954607737984, 1.9399996262585357 ], [ -1.3199945576212193, -0.4279939890191314, 1.2159985379893223 ], [ -1.6519921530643058, -1.5719908174912227, -0.5319998347887945 ], [ -0.28000097322811607, -1.1079990676093765, -0.5519965596163635 ], [ 1.2639927769962411, 0.008003575330566675, 0.5279975119066194 ], [ -0.2439982266435559, -1.1239949321857712, 1.1119969314963016 ], [ 1.5199965703924747, -1.4199997723935254, 0.016001813161774122 ], ... ],
[ [ -1.8919930236797033, -1.519994704167697, 0.49200228992749995 ], [ 0.2119983945392579, 1.6119975017511772, -0.556000124993985 ], [ 1.699995565039036, 1.4359982065352026, -1.4039948080210753 ], [ 0.3279957205060117, 1.8719928377394492, -1.447997015653232 ], [ 1.511993813443608, -0.9799935640760004, -1.5999993372947343 ], [ -0.8919978327936502, -1.4280004771770236, -0.3280006224888006 ], [ 0.6920004777977681, 0.6879974810221352, 1.819998133774993 ], [ -1.9719941637712537, 1.055995501408612, 1.1759936666886956 ], ... ],
[ [ -1.3959948270975817, 1.6999919671243264, -1.8959958548323999 ], [ 0.22799858109975837, -1.7920002698878184, 1.8839958950495026 ], [ -1.139992961492723, 1.5319956154176846, 0.319994578761549 ], [ -0.6440001647994321, 1.835998849132173, -0.1440031099617109 ], [ 1.1319952126469248, -0.6399935951691879, -1.203999223285583 ], [ -0.12399863085789845, -1.1599979171533448, 0.33200274596392876 ], [ 1.9319915228958588, -0.25199925416967894, 0.9960022070117718 ], [ 1.7519974513724053, -1.8759985804791282, 0.5560011601999726 ], ... ],
[ [ 1.2439955650390337, -1.231993615898126, -0.7400019076831565 ], [ -1.5199928785771202, -0.9119943517754017, -1.4039973680451223 ], [ -1.8719989003338222, 1.4239994710005126, 1.459996589468855 ], [ -1.0639963506743886, 0.6480002379712436, -0.687993263715111 ], [ 0.7399960625335334, -0.019999347449998348, -0.31200003171386326 ], [ 0.38800233303759774, -0.8720021769504714, -0.2519997829665778 ], [ 1.5719930050147175, -1.5119927452827286, 0.6560001652111305 ], [ 1.9519978244936826, -0.016003513967158326, -1.8239987776128481 ], ... ],
[ [ 1.1319926318938704, -1.1479920715922065, -1.6839910768118616 ], [ -1.2439982473725093, 1.1359939674668358, 1.9599961437966849 ], [ -0.25999431923843647, -0.5359985493858873, 0.8279951902654926 ], [ -1.7839954489656764, -0.25599932672103937, -0.7680030270460328 ], [ 0.9359970678871525, 0.4240024870610437, 0.9999984550735461 ], [ -0.612002714459093, 1.5559909721354235, 1.576000279220225 ], [ 0.19600364932530026, -1.6639961033708288, 0.5039992945955492 ], [ 1.6239964563834062, -1.6519976165834231, -0.1079973576806841 ], ... ],
[ [ 1.727996642943808, -1.5919984353768184, -0.6479993269298769 ], [ 0.5919970471581137, -0.3400011197742289, 1.7279920291020836 ], [ 1.4000004881624601, -0.6320022598662225, 0.42000284960868595 ], [ -1.3479960708338388, -1.5760009332139393, -0.6519975131478397 ], [ 0.35599503645129676, -0.46400157581112794, -0.12000417750213928 ], [ -0.7759992734549632, 0.46000220722058405, -1.9479951707775551 ], [ 0.4480029756350151, 0.7000023834164932, -0.5919981350158684 ], [ -0.9799939150238204, -1.8959958442589233, -0.6839960413929602 ], ... ],
[ [ -1.1239991801748623, -0.20799871521731825, -1.0600021253364351 ], [ 0.032003307297765474, -1.511991501546156, -0.11200425005335007 ], [ 0.8679960418045621, -0.7279986530306204, 1.6519935112213975 ], [ 0.3599991822389395, -0.7839966837811901, -0.7879981972028239 ], [ 0.9840018977298023, -1.0639979586110402, 0.9399964650952163 ], [ 1.5679961454494582, 1.3119945271480695, 1.2959922778493929 ], [ 0.41999374089231956, 0.6760022486784367, 1.7759968382161797 ], [ -1.1079974078505148, 1.503996123276613, 1.151993127735986 ], ... ],
...
]
To produce the following output:
TrainingTester.java:633 executed in 0.14 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ 1.127444863319397, -0.1717802882194519, -1.847672462463379 ], [ 0.8474469184875488, -1.975777506828308, -1.1156706809997559 ], [ -1.0845469236373901, -0.9517812728881836, -2.003668785095215 ], [ 1.0714517831802368, -0.08378028869628906, -1.7956674098968506 ], [ 0.9274460077285767, 0.9442120790481567, -2.463669538497925 ], [ 0.9874453544616699, 0.5442150831222534, 0.4043232202529907 ], [ 0.38745009899139404, -2.5157740116119385, -1.8356680870056152 ], [ -0.5365504026412964, 0.7762134075164795, 0.8163233995437622 ], ... ],
[ [ 0.9794481992721558, -2.67977237701416, -0.8356753587722778 ], [ -0.6885420083999634, -2.231776714324951, 1.1083292961120605 ], [ -0.8805429935455322, -1.2317752838134766, 0.3843282461166382 ], [ -1.212540626525879, -2.375771999359131, -1.3636701107025146 ], [ 0.15945056080818176, -1.9117803573608398, -1.3836668729782104 ], [ 1.7034443616867065, -0.7957776784896851, -0.30367279052734375 ], [ 0.19545331597328186, -1.9277762174606323, 0.28032660484313965 ], [ 1.9594480991363525, -2.223781108856201, -0.8156684637069702 ], ... ],
[ [ -1.452541470527649, -2.3237760066986084, -0.33966800570487976 ], [ 0.6514499187469482, 0.8082162141799927, -1.3876703977584839 ], [ 2.139446973800659, 0.6322169303894043, -2.2356650829315186 ], [ 0.7674472332000732, 1.068211555480957, -2.279667377471924 ], [ 1.9514453411102295, -1.7837748527526855, -2.4316697120666504 ], [ -0.45254626870155334, -2.2317817211151123, -1.1596709489822388 ], [ 1.131451964378357, -0.11578381061553955, 0.9883278608322144 ], [ -1.5325425863265991, 0.2522141933441162, 0.34432339668273926 ], ... ],
[ [ -0.9565432071685791, 0.8962106704711914, -2.727666139602661 ], [ 0.6674501299858093, -2.5957815647125244, 1.0523256063461304 ], [ -0.7005413770675659, 0.7282143831253052, -0.5116757154464722 ], [ -0.20454862713813782, 1.0322176218032837, -0.9756733775138855 ], [ 1.5714467763900757, -1.443774938583374, -2.0356695652008057 ], [ 0.31545290350914, -1.9637792110443115, -0.49966752529144287 ], [ 2.371443033218384, -1.0557805299758911, 0.16433191299438477 ], [ 2.191448926925659, -2.6797800064086914, -0.2756690979003906 ], ... ],
[ [ 1.6834471225738525, -2.0357749462127686, -1.5716722011566162 ], [ -1.0805412530899048, -1.7157756090164185, -2.2356677055358887 ], [ -1.4325473308563232, 0.6202181577682495, 0.6283262968063354 ], [ -0.6245447397232056, -0.15578103065490723, -1.5196635723114014 ], [ 1.1794476509094238, -0.8237805962562561, -1.1436703205108643 ], [ 0.8274538516998291, -1.675783395767212, -1.0836701393127441 ], [ 2.011444568634033, -2.3157739639282227, -0.17567014694213867 ], [ 2.391449451446533, -0.8197847604751587, -2.6556692123413086 ], ... ],
[ [ 1.5714441537857056, -1.9517732858657837, -2.5156612396240234 ], [ -0.8045467138290405, 0.3322126865386963, 1.1283258199691772 ], [ 0.17945721745491028, -1.3397798538208008, -0.003675103187561035 ], [ -1.3445438146591187, -1.0597805976867676, -1.5996732711791992 ], [ 1.3754485845565796, -0.37977877259254456, 0.1683281660079956 ], [ -0.17255118489265442, 0.7522096633911133, 0.7443300485610962 ], [ 0.6354551911354065, -2.4677772521972656, -0.32767099142074585 ], [ 2.063447952270508, -2.4557788372039795, -0.9396676421165466 ], ... ],
[ [ 2.167448043823242, -2.395779609680176, -1.4796695709228516 ], [ 1.0314486026763916, -1.143782377243042, 0.8963217735290527 ], [ 1.8394520282745361, -1.4357835054397583, -0.41166743636131287 ], [ -0.9085445404052734, -2.379782199859619, -1.4836678504943848 ], [ 0.7954465746879578, -1.2677828073501587, -0.9516744613647461 ], [ -0.33654770255088806, -0.343779057264328, -2.779665470123291 ], [ 0.8874545097351074, -0.1037788987159729, -1.423668384552002 ], [ -0.5405423641204834, -2.699777126312256, -1.5156662464141846 ], ... ],
[ [ -0.6845476627349854, -1.011780023574829, -1.8916723728179932 ], [ 0.4714548587799072, -2.315772771835327, -0.9436745643615723 ], [ 1.3074475526809692, -1.5317800045013428, 0.8203232288360596 ], [ 0.7994507551193237, -1.5877779722213745, -1.6196684837341309 ], [ 1.423453450202942, -1.86777925491333, 0.10832619667053223 ], [ 2.0074477195739746, 0.5082132816314697, 0.46432197093963623 ], [ 0.8594452738761902, -0.1277790069580078, 0.9443265199661255 ], [ -0.6685458421707153, 0.7002148628234863, 0.3203228712081909 ], ... ],
...
]
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 46.63 seconds (2.537 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 5407180740216
Reset training subject: 5408091104653
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=3.298985639906153}, derivative=-0.16864947480671733}
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=3.298985639906153}, derivative=-0.16864947480671733}, evalInputDelta = 0.0
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=3.298985639906153}, derivative=-0.16864947480671733}, evalInputDelta = 0.0
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=3.298985639906153}, derivative=-0.16864947480671733}, evalInputDelta = 0.0
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=3.298985639906153}, derivative=-0.16864947480671733}, evalInputDelta = 0.0
New Minimum: 3.298985639906153 > 3.298985594793764
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=3.298985594793764}, derivative=-0.16864947080914217}, evalInputDelta = -4.511238893556424E-8
New Minimum: 3.298985594793764 > 3.2989853550035044
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=3.2989853550035044}, derivative=-0.16864943290781345}, evalInputDelta = -2.849026485840511E-7
New Minimum: 3.2989853550035044 > 3.298983645932617
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=3.298983645932617}, derivative=-0.16864920696532412}, evalInputDelta = -1.9939735360630095E-6
New Minimum: 3.298983645932617 > 3.298971739898954
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=3.298971739898954}, derivative=-0.16864762377438344}, evalInputDelta = -1.3900007199030284E-5
New Minimum: 3.298971739898954 > 3.2988884269867924
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=3.2988884269867924}, derivative=-0.16863650964103843}, evalInputDelta = -9.721291936060084E-5
New Minimum: 3.2988884269867924 > 3.2983052638012986
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=3.2983052638012986}, derivative=-0.16855873443642583}, evalInputDelta = -6.803761048543855E-4
New Minimum: 3.2983052638012986 > 3.294230667588952
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=3.294230667588952}, derivative=-0.16801427992582996}, evalInputDelta = -0.004754972317201123
New Minimum: 3.294230667588952 > 3.266077705501439
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=3.266077705501439}, derivative=-0.164203146396369}, evalInputDelta = -0.032907934404713846
New Minimum: 3.266077705501439 > 3.087093051404452
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=3.087093051404452}, derivative=-0.13752519601025048}, evalInputDelta = -0.2118925885017009
New Minimum: 3.087093051404452 > 2.7204176440760324
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=2.7204176440760324}, derivative=0.049220450406351215}, evalInputDelta = -0.5785679958301206
2.7204176440760324 <= 3.298985639906153
New Minimum: 2.7204176440760324 > 2.666548705942329
F(7.500016674491876) = LineSearchPoint{point=PointSample{avg=2.666548705942329}, derivative=-4.194080892841247E-10}, evalInputDelta = -0.6324369339638238
Left bracket at 7.500016674491876
Converged to left
Fitness changed from 3.298985639906153 to 2.666548705942329
Iteration 1 complete. Error: 2.666548705942329 Total: 26.8993; Orientation: 0.6962; Line Search: 23.7738
F(0.0) = LineSearchPoint{point=PointSample{avg=2.666548705942329}, derivative=-4.938067595379742E-7}
New Minimum: 2.666548705942329 > 2.66654500246703
F(7.500016674491876) = LineSearchPoint{point=PointSample{avg=2.66654500246703}, derivative=-4.938049777255074E-7}, evalInputDelta = -3.703475299321468E-6
New Minimum: 2.66654500246703 > 2.6665227782585896
F(52.500116721443135) = LineSearchPoint{point=PointSample{avg=2.6665227782585896}, derivative=-4.937942561533582E-7}, evalInputDelta = -2.5927683739546126E-5
New Minimum: 2.6665227782585896 > 2.6663672477102645
F(367.50081705010194) = LineSearchPoint{point=PointSample{avg=2.6663672477102645}, derivative=-4.937192218656664E-7}, evalInputDelta = -1.81458232064724E-4
New Minimum: 2.6663672477102645 > 2.6652791718940754
F(2572.5057193507137) = LineSearchPoint{point=PointSample{avg=2.6652791718940754}, derivative=-4.931939744475185E-7}, evalInputDelta = -0.0012695340482538064
New Minimum: 2.6652791718940754 > 2.657695082522852
F(18007.540035454997) = LineSearchPoint{point=PointSample{avg=2.657695082522852}, derivative=-4.895172585984918E-7}, evalInputDelta = -0.008853623419477152
New Minimum: 2.657695082522852 > 2.606195454479184
F(126052.78024818498) = LineSearchPoint{point=PointSample{avg=2.606195454479184}, derivative=-4.637802298618048E-7}, evalInputDelta = -0.060353251463145074
New Minimum: 2.606195454479184 > 2.3235594242209805
F(882369.4617372949) = LineSearchPoint{point=PointSample{avg=2.3235594242209805}, derivative=-2.83621054752804E-7}, evalInputDelta = -0.3429892817213487
F(6176586.232161065) = LineSearchPoint{point=PointSample{avg=4.160314179961867}, derivative=9.774931932433699E-7}, evalInputDelta = 1.493765474019538
F(475122.01785854343) = LineSearchPoint{point=PointSample{avg=2.458816660899129}, derivative=-3.806298426448773E-7}, evalInputDelta = -0.2077320450432003
F(3325854.125009804) = LineSearchPoint{point=PointSample{avg=2.341654213527559}, derivative=2.984316627781433E-7}, evalInputDelta = -0.3248944924147703
2.341654213527559 <= 2.666548705942329
New Minimum: 2.3235594242209805 > 2.1547120913747952
F(2073023.9810456668) = LineSearchPoint{point=PointSample{avg=2.1547120913747952}, derivative=9.130212329751148E-15}, evalInputDelta = -0.5118366145675339
Right bracket at 2073023.9810456668
Converged to right
Fitness changed from 2.666548705942329 to 2.1547120913747952
Iteration 2 complete. Error: 2.1547120913747952 Total: 19.7299; Orientation: 0.6817; Line Search: 18.2839
Final threshold in iteration 2: 2.1547120913747952 (> 0.0) after 46.630s (< 30.000s)
Returns
2.1547120913747952
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.01 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.1928808550810518, 0.7025948290060424, -1.1811375881682704 ], [ 0.06336745503165897, -0.8925665928361732, -0.318613439636611 ], [ -1.415637094483994, -0.1472739568973262, -1.050752291291165 ], [ 0.6497638199954356, 0.7944337839730439, -0.7521532057708429 ], [ 0.07656981021998299, 1.2542699819432999, -1.5806211000766075 ], [ 0.09050249934828858, 1.0738576225676093, 0.7583723470295529 ], [ -0.16015355969391373, -1.1699828053136878, -0.8405239239456006 ], [ -1.1233107745069142, 1.1876181184894832, 1.183424759594495 ], ... ],
[ [ 0.29518000959104157, -1.2072976231899837, -0.37951171398995104 ], [ -0.6495628653500725, -1.091750219497912, 1.9122966449811287 ], [ -0.9168061538843808, 0.01727580410044799, 1.1076788309339318 ], [ -1.0706791239353048, -0.8917807697300997, -0.5197783148895253 ], [ -0.3520674859725854, -1.0389407529807166, -0.29715700166797865 ], [ 0.7289557920698805, 0.2728454240529141, 0.34366771708331956 ], [ -0.11260342474703672, -0.7485929818029208, 0.8846715473032766 ], [ 1.265966551993018, -1.403150403240733, 0.15030012127369416 ], ... ],
[ [ -1.375173384325826, -1.1277016371528856, 0.6616184211629914 ], [ 0.0930974007213037, 1.4269211375422508, -0.5652763940591732 ], [ 1.3714910372905964, 1.30313077636123, -1.019400678452582 ], [ 0.011007850707472122, 1.341416094346725, -1.2269397138501223 ], [ 1.0537347083548885, -0.5032448452567376, -1.5509244844975971 ], [ -0.7314274442241846, -1.4633600077368498, -0.37413029309666085 ], [ 0.7274229556389535, 0.5013855441144961, 1.6817350737187893 ], [ -1.5396301647808543, 0.7227378762490646, 0.7068180593979099 ], ... ],
[ [ -1.0127686739490536, 1.1049218453971168, -1.588947397171696 ], [ 0.12291759283877456, -1.8120042793961466, 1.5798926767821264 ], [ -0.6185666612472085, 1.207183542853688, -0.08161655056083061 ], [ -0.6561799486605784, 1.7507336577788952, -0.37439962811127875 ], [ 0.777386206440235, -0.16554820679598425, -1.1464787721294056 ], [ -0.022547196263268454, -1.0057161700913337, 0.5354011161836213 ], [ 1.3040533564510999, -0.1967609450981875, 1.1594761103772204 ], [ 1.5632286033770222, -1.7708546563775005, 0.6419294145398079 ], ... ],
[ [ 0.915491031570202, -0.7590837659009593, -0.8813361016957246 ], [ -0.9924243561819562, -0.493596800581882, -1.2090445389800064 ], [ -1.790509676284664, 1.3848010144143648, 1.20733451316671 ], [ -0.7936336181915529, 0.6656173991937043, -0.1890001751051909 ], [ 0.44834489628491, 0.028328947402164288, -0.3143663094413505 ], [ 0.560857165906403, -1.0332774103080247, -0.23593937136356086 ], [ 1.0538471824836493, -0.9745895055104383, 0.6682216965513048 ], [ 1.790868987611964, -0.276322208196828, -1.7334638394997077 ], ... ],
[ [ 0.586206741044185, -0.5606832625534852, -1.0229966759307998 ], [ -1.114139015314553, 0.6891051066903027, 1.6743195805371363 ], [ 0.16085298747951088, -0.42855130627920823, 0.4716830150812802 ], [ -1.4468360077026152, -0.206135459170081, -0.9922573230459437 ], [ 0.718820402406699, 0.6082275756828752, 0.8855365201481195 ], [ -0.813056029635515, 0.8872141388496094, 1.596667363155928 ], [ 0.4663663474856261, -1.3753531009155358, 0.45172741327891686 ], [ 1.3615208739476063, -1.475450288487497, 0.0877232291315269 ], ... ],
[ [ 1.4793410660649944, -1.476105651115622, -0.5981566591814999 ], [ 0.3732848461618797, -0.4229629349181244, 1.1375073799051454 ], [ 1.4361907809552164, -0.7994197382551439, 0.6310790089562339 ], [ -1.056903330015388, -1.645142765682479, -0.4677936128876101 ], [ -0.01166618324279839, -0.5807456699820938, -0.4294818830535837 ], [ -0.7221501062832153, 0.6234972932272081, -1.5902732831350777 ], [ 0.6684600930878019, 0.8765497000717143, -0.4538609352002586 ], [ -0.5292032267030233, -1.5881583832851345, -0.39076758928925 ], ... ],
[ [ -1.06323999878365, -0.11283592785074992, -1.2174596553523593 ], [ 0.2770293133490597, -0.8824548608849756, -0.42685640856643137 ], [ 0.5748093028572381, -0.6282292047722566, 1.17130120593786 ], [ 0.29944935032605385, -0.538349293577363, -0.6544676811606603 ], [ 1.1246100128292755, -0.9127873569466778, 0.678121044127815 ], [ 1.2824871899094152, 0.9065657059244583, 0.723934317983018 ], [ -0.04363981715804216, 0.8425684514803322, 1.5417614398036281 ], [ -0.9159480993018473, 1.216805198561796, 0.6428930192686465 ], ... ],
...
]
To produce the following output:
TrainingTester.java:633 executed in 0.15 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ 0.6357443332672119, -0.10819840431213379, -2.0257692337036133 ], [ 0.5062309503555298, -1.703359842300415, -1.1632450819015503 ], [ -0.9727736711502075, -0.9580671787261963, -1.8953839540481567 ], [ 1.0926272869110107, -0.016359448432922363, -1.5967848300933838 ], [ 0.519433319568634, 0.44347673654556274, -2.425252914428711 ], [ 0.5333659648895264, 0.263064444065094, -0.08625930547714233 ], [ 0.2827099561691284, -1.980776071548462, -1.6851556301116943 ], [ -0.6804473400115967, 0.37682491540908813, 0.3387930393218994 ], ... ],
[ [ 0.7380435466766357, -2.0180909633636475, -1.224143385887146 ], [ -0.20669934153556824, -1.902543544769287, 1.0676649808883667 ], [ -0.47394266724586487, -0.7935174107551575, 0.2630472183227539 ], [ -0.6278156042098999, -1.7025740146636963, -1.3644099235534668 ], [ 0.09079599380493164, -1.8497340679168701, -1.1417887210845947 ], [ 1.1718193292617798, -0.5379477739334106, -0.5009639263153076 ], [ 0.3302600681781769, -1.5593862533569336, 0.040039896965026855 ], [ 1.7088299989700317, -2.2139437198638916, -0.6943315267562866 ], ... ],
[ [ -0.9323098659515381, -1.9384949207305908, -0.18301326036453247 ], [ 0.5359609127044678, 0.6161279082298279, -1.4099080562591553 ], [ 1.8143545389175415, 0.49233752489089966, -1.8640323877334595 ], [ 0.45387133955955505, 0.5306228995323181, -2.0715713500976562 ], [ 1.4965981245040894, -1.3140380382537842, -2.3955562114715576 ], [ -0.28856393694877625, -2.274153232574463, -1.218761920928955 ], [ 1.1702864170074463, -0.30940765142440796, 0.8371033668518066 ], [ -1.096766710281372, -0.08805537223815918, -0.13781362771987915 ], ... ],
[ [ -0.569905161857605, 0.2941285967826843, -2.4335789680480957 ], [ 0.5657811164855957, -2.6227974891662598, 0.7352609634399414 ], [ -0.17570313811302185, 0.3963903784751892, -0.9262481927871704 ], [ -0.21331647038459778, 0.9399403929710388, -1.2190313339233398 ], [ 1.2202496528625488, -0.9763414263725281, -1.9911104440689087 ], [ 0.42031630873680115, -1.816509485244751, -0.3092305660247803 ], [ 1.7469167709350586, -1.0075541734695435, 0.3148444890975952 ], [ 2.006092071533203, -2.5816478729248047, -0.20270228385925293 ], ... ],
[ [ 1.3583545684814453, -1.5698769092559814, -1.7259677648544312 ], [ -0.5495609045028687, -1.3043899536132812, -2.053676128387451 ], [ -1.3476462364196777, 0.5740078091621399, 0.3627028465270996 ], [ -0.35077014565467834, -0.14517581462860107, -1.0336318016052246 ], [ 0.8912084102630615, -0.7824642658233643, -1.1589980125427246 ], [ 1.0037206411361694, -1.8440706729888916, -1.0805710554122925 ], [ 1.4967106580734253, -1.7853827476501465, -0.17640995979309082 ], [ 2.2337324619293213, -1.0871154069900513, -2.5780954360961914 ], ... ],
[ [ 1.029070258140564, -1.371476411819458, -1.8676283359527588 ], [ -0.6712754964828491, -0.1216881275177002, 0.8296879529953003 ], [ 0.6037164926528931, -1.239344596862793, -0.37294864654541016 ], [ -1.0039725303649902, -1.0169286727905273, -1.8368890285491943 ], [ 1.1616839170455933, -0.20256567001342773, 0.04090481996536255 ], [ -0.3701925575733185, 0.07642090320587158, 0.7520357370376587 ], [ 0.9092298746109009, -2.1861462593078613, -0.39290425181388855 ], [ 1.8043843507766724, -2.286243438720703, -0.7569084167480469 ], ... ],
[ [ 1.9222044944763184, -2.2868988513946533, -1.4427883625030518 ], [ 0.8161483407020569, -1.233756184577942, 0.2928757667541504 ], [ 1.8790541887283325, -1.610213041305542, -0.21355265378952026 ], [ -0.6140398979187012, -2.4559359550476074, -1.3124252557754517 ], [ 0.43119731545448303, -1.3915388584136963, -1.2741135358810425 ], [ -0.27928659319877625, -0.18729591369628906, -2.4349050521850586 ], [ 1.111323595046997, 0.06575649976730347, -1.298492670059204 ], [ -0.08633974194526672, -2.398951530456543, -1.2353992462158203 ], ... ],
[ [ -0.6203765869140625, -0.9236291646957397, -2.06209135055542 ], [ 0.7198927998542786, -1.6932480335235596, -1.271488070487976 ], [ 1.0176727771759033, -1.439022421836853, 0.3266695737838745 ], [ 0.7423128485679626, -1.3491425514221191, -1.4990993738174438 ], [ 1.5674735307693481, -1.7235805988311768, -0.16651064157485962 ], [ 1.7253506183624268, 0.09577250480651855, -0.12069737911224365 ], [ 0.3992236852645874, 0.03177523612976074, 0.6971297264099121 ], [ -0.47308459877967834, 0.40601199865341187, -0.2017386555671692 ], ... ],
...
]
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 48.31 seconds (1.693 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 5454211433723
Reset training subject: 5454958659787
Adding measurement 60b0ef7d to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 3.298985639906153 < 3.298985639906153. Total: 1
th(0)=3.298985639906153;dx=-0.1686494748067173
Adding measurement 3d874fa1 to history. Total: 1
New Minimum: 3.298985639906153 > 2.9878280034704936
END: th(2.154434690031884)=2.9878280034704936; dx=-0.12020367707852045 evalInputDelta=0.3111576364356594
Fitness changed from 3.298985639906153 to 2.9878280034704936
Iteration 1 complete. Error: 2.9878280034704936 Total: 6.3896; Orientation: 0.8816; Line Search: 3.2651
Non-optimal measurement 2.9878280034704936 < 2.9878280034704936. Total: 2
LBFGS Accumulation History: 2 points
Non-optimal measurement 2.9878280034704936 < 2.9878280034704936. Total: 2
th(0)=2.9878280034704936;dx=-0.08567433057099197
Adding measurement 18b398b7 to history. Total: 2
New Minimum: 2.9878280034704936 > 2.7132155352032257
END: th(4.641588833612779)=2.7132155352032257; dx=-0.032652596772015145 evalInputDelta=0.27461246826726793
Fitness changed from 2.9878280034704936 to 2.7132155352032257
Iteration 2 complete. Error: 2.7132155352032257 Total: 5.8261; Orientation: 0.7184; Line Search: 4.3530
Non-optimal measurement 2.7132155352032257 < 2.7132155352032257. Total: 3
LBFGS Accumulation History: 3 points
Non-optimal measurement 2.7132155352032257 < 2.7132155352032257. Total: 3
th(0)=2.7132155352032257;dx=-0.01244489770259318
Adding measurement 776d71a6 to history. Total: 3
New Minimum: 2.7132155352032257 > 2.6717293049407016
WOLF (strong): th(10.000000000000002)=2.6717293049407016; dx=0.004147652549703677 evalInputDelta=0.04148623026252407
Non-optimal measurement 2.6717317364913913 < 2.6717293049407016. Total: 4
END: th(5.000000000000001)=2.6717317364913913; dx=-0.004148622844924993 evalInputDelta=0.041483798711834385
Fitness changed from 2.7132155352032257 to 2.6717293049407016
Iteration 3 complete. Error: 2.6717293049407016 Total: 6.2855; Orientation: 0.7393; Line Search: 4.7708
Non-optimal measurement 2.6717293049407016 < 2.6717293049407016. Total: 4
Rejected: LBFGS Orientation magnitude: 2.789e-01, gradient 3.719e-02, dot -1.000; [2f39023d-2932-45d6-a654-b8d1e9983117 = 1.000/1.000e+00, ad994b20-44b2-4815-abe2-819680de6574 = 1.000/1.000e+00, 4afa1b65-b7eb-4c25-a26b-92e766fcefd3 = 1.000/1.000e+00, 49a5664f-1f5b-405e-a1cd-45506007dac9 = 1.000/1.000e+00, 8eba8e2e-ad2e-4ab6-870c-3ccc21e8186c = 1.000/1.000e+00, c0ab33a9-fa25-404f-ac16-4c59a892ecf5 = 1.000/1.000e+00, 22d5cea2-cc57-4332-8d04-05bea3d95171 = 1.000/1.000e+00, 9dde5276-f7c5-492e-9eb1-ab7d0a693705 = 1.000/1.000e+00, 173f7be0-05a0-400c-a703-eed138efb8f1 = 1.000/1.000e+00, 072d321e-d5a9-4164-8857-7ab224add3a4 = 1.000/1.000e+00]
Orientation rejected. Popping history element from 2.6717293049407016, 2.7132155352032257, 2.9878280034704936, 3.298985639906153
LBFGS Accumulation History: 3 points
Removed measurement 776d71a6 to history. Total: 3
Adding measurement 1152ab9c to history. Total: 3
th(0)=2.6717293049407016;dx=-0.0013832131634255664
Adding measurement 457872ed to history. Total: 4
New Minimum: 2.6717293049407016 > 2.6675257917945885
WOLF (strong): th(10.772173450159421)=2.6675257917945885; dx=6.027739076559368E-4 evalInputDelta=0.004203513146113114
Adding measurement 58cb970 to history. Total: 5
New Minimum: 2.6675257917945885 > 2.666953379661158
END: th(5.386086725079711)=2.666953379661158; dx=-3.9021957642869473E-4 evalInputDelta=0.004775925279543625
Fitness changed from 2.6717293049407016 to 2.666953379661158
Iteration 4 complete. Error: 2.666953379661158 Total: 29.8123; Orientation: 23.6314; Line Search: 5.4177
Final threshold in iteration 4: 2.666953379661158 (> 0.0) after 48.314s (< 30.000s)
Returns
2.666953379661158
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.6879947016759624, 0.6320007555425314, -1.0160017667922083 ], [ 0.4079963119314882, -1.171997009816685, -0.28400037014146795 ], [ -1.5239988408700174, -0.14799999207620482, -1.171998702376509 ], [ 0.6320001896903781, 0.7200007966205116, -0.9639977329368569 ], [ 0.4879955971749593, 1.7479947170834724, -1.631999449995419 ], [ 0.5479951042398933, 1.3479970667423704, 1.2359948894626291 ], [ -0.05200115766662172, -1.7119942000848987, -1.0039982505190994 ], [ -0.9760015766614833, 1.5799958015413913, 1.6479950291276835 ], ... ],
[ [ 0.5399973799583212, -1.8759928445123542, -0.004004017863753895 ], [ -1.1279948809555875, -1.4279964018631102, 1.939999703798784 ], [ -1.3199956860832989, -0.427995235249333, 1.2159988411619216 ], [ -1.6519937800663693, -1.5719927212785494, -0.531999868990278 ], [ -0.28000077153388936, -1.1079992608887126, -0.5519972728636605 ], [ 1.263994274464693, 0.008002834086981119, 0.5279980278182734 ], [ -0.24399859440234734, -1.1239959828679587, 1.1119975677452076 ], [ 1.5199972813710616, -1.4199998195489847, 0.01600143728849582 ], ... ],
[ [ -1.891994470175937, -1.519995802125262, 0.4920018152057385 ], [ 0.21199872731511274, 1.6119980197508477, -0.5560000990268585 ], [ 1.6999964844589175, 1.4359985784110925, -1.403995884428952 ], [ 0.3279966076927162, 1.8719943227348785, -1.447997634349966 ], [ 1.511995096024085, -0.9799948984102608, -1.599999474642333 ], [ -0.8919982822098682, -1.4280003782088624, -0.3280004933754308 ], [ 0.6920003786488812, 0.6879980033196139, 1.819998520753843 ], [ -1.9719953738907599, 1.0559964341419836, 1.175994979834495 ], ... ],
[ [ -1.3959958996887636, 1.6999936326255534, -1.8959967142035736 ], [ 0.227998875195788, -1.7920002138971625, 1.8839967461862874 ], [ -1.139994420882221, 1.5319965245133835, 0.3199957028060625 ], [ -0.6440001307177867, 1.8359990877775787, -0.14400246511725268 ], [ 1.1319962051286967, -0.639994923056816, -1.2039993842708543 ], [ -0.12399891481039121, -1.1599983489582644, 0.3320021766915243 ], [ 1.931993280378679, -0.25199940876922183, 0.9960017494810348 ], [ 1.7519979796962744, -1.8759988747558516, 0.5560009197061306 ], ... ],
[ [ 1.2439964844589313, -1.231994939488042, -0.7400015121086903 ], [ -1.519994355157659, -0.911995522794873, -1.403997913679854 ], [ -1.8719991284155453, 1.423999580713114, 1.459997296630747 ], [ -1.0639971073804004, 0.6480001886668001, -0.6879946603060618 ], [ 0.7399968788072196, -0.019999482709612614, -0.3120000250866334 ], [ 0.38800184923937864, -0.8720017255660111, -0.2519998279124696 ], [ 1.571994455207856, -1.5119942493784744, 0.656000131009647 ], [ 1.9519982754578373, -0.016002785377225055, -1.8239990310000826 ], ... ],
[ [ 1.1319941594466274, -1.1479937153650688, -1.6839929268164349 ], [ -1.2439986108335725, 1.1359952182344257, 1.9599969433603799 ], [ -0.2599954971247177, -0.5359988501091468, 0.82799618752609 ], [ -1.7839963926241182, -0.2559994662784298, -0.7680023993926152 ], [ 0.9359976757195682, 0.4240019714498254, 0.999998775437138 ], [ -0.6120021517531269, 1.555992843929009, 1.5760002213810689 ], [ 0.19600289261938447, -1.6639969112296564, 0.5039994408999274 ], [ 1.6239971909996977, -1.651998110705937, -0.10799790546430386 ], ... ],
[ [ 1.7279973388802425, -1.5919987597377827, -0.6479994664264598 ], [ 0.5919976592882975, -0.3400008875755195, 1.7279936817709178 ], [ 1.4000003868645892, -0.6320017912907215, 0.4200022588474808 ], [ -1.3479968855595084, -1.5760007396949591, -0.6519980286982665 ], [ 0.35599606546400575, -0.4640012490616635, -0.12000331132313802 ], [ -0.7759994241769558, 0.4600017496290555, -1.9479961719747014 ], [ 0.44800235860621573, 0.7000018892940435, -0.591998521633587 ], [ -0.9799951767164483, -1.8959967058397926, -0.6839968620842767 ], ... ],
[ [ -1.1239993502367676, -0.20799898155852867, -1.0600016846356715 ], [ 0.03200262150488395, -1.511993263507446, -0.11200336883219635 ], [ 0.8679968623760566, -0.7279989322650952, 1.651994856600422 ], [ 0.3599993516999892, -0.7839973713027659, -0.7879985709272141 ], [ 0.984001504184498, -1.0639983818204717, 0.9399971980436693 ], [ 1.567996944532136, 1.3119956618761517, 1.2959938789451055 ], [ 0.4199950385149811, 0.676001782491404, 1.7759974938049823 ], [ -1.1079979453707596, 1.5039969270772766, 1.151994552623581 ], ... ],
...
]
To produce the following output:
TrainingTester.java:633 executed in 0.15 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ 1.1274354457855225, -0.17177629470825195, -1.8476656675338745 ], [ 0.8474370837211609, -1.9757740497589111, -1.1156642436981201 ], [ -1.084558129310608, -0.9517770409584045, -2.003662586212158 ], [ 1.0714409351348877, -0.08377623558044434, -1.7956615686416626 ], [ 0.927436351776123, 0.9442176222801208, -2.463663339614868 ], [ 0.9874358177185059, 0.5442200303077698, 0.4043310880661011 ], [ 0.3874396085739136, -2.5157711505889893, -1.8356621265411377 ], [ -0.5365607738494873, 0.7762187123298645, 0.816331148147583 ], ... ],
[ [ 0.9794381856918335, -2.679769992828369, -0.8356678485870361 ], [ -0.6885541677474976, -2.2317733764648438, 1.1083358526229858 ], [ -0.8805549144744873, -1.2317723035812378, 0.3843350410461426 ], [ -1.2125530242919922, -2.375769853591919, -1.363663673400879 ], [ 0.15943998098373413, -1.9117763042449951, -1.3836610317230225 ], [ 1.703434944152832, -0.7957742214202881, -0.30366581678390503 ], [ 0.19544216990470886, -1.9277729988098145, 0.2803337574005127 ], [ 1.9594379663467407, -2.2237768173217773, -0.8156623840332031 ], ... ],
[ [ -1.4525537490844727, -2.32377290725708, -0.3396620452404022 ], [ 0.6514394879341125, 0.8082209229469299, -1.3876639604568481 ], [ 2.139437198638916, 0.632221519947052, -2.235659599304199 ], [ 0.7674373388290405, 1.0682172775268555, -2.2796616554260254 ], [ 1.9514358043670654, -1.7837719917297363, -2.4316632747650146 ], [ -0.4525575339794159, -2.2317774295806885, -1.1596643924713135 ], [ 1.1314411163330078, -0.11577904224395752, 0.9883346557617188 ], [ -1.5325546264648438, 0.25221937894821167, 0.34433114528656006 ], ... ],
[ [ -0.9565551280975342, 0.8962165713310242, -2.727660655975342 ], [ 0.6674396395683289, -2.5957772731781006, 1.052332878112793 ], [ -0.7005536556243896, 0.7282194495201111, -0.5116681456565857 ], [ -0.2045593559741974, 1.032222032546997, -0.9756662845611572 ], [ 1.571436882019043, -1.4437719583511353, -2.03566312789917 ], [ 0.3154418468475342, -1.963775396347046, -0.4996616840362549 ], [ 2.371433973312378, -1.0557764768600464, 0.1643378734588623 ], [ 2.191438674926758, -2.6797759532928467, -0.275662899017334 ], ... ],
[ [ 1.6834372282028198, -2.0357720851898193, -1.5716652870178223 ], [ -1.080553650856018, -1.7157726287841797, -2.235661745071411 ], [ -1.4325584173202515, 0.6202225089073181, 0.6283334493637085 ], [ -0.6245564222335815, -0.15577685832977295, -1.5196585655212402 ], [ 1.1794376373291016, -0.8237765431404114, -1.1436638832092285 ], [ 0.8274426460266113, -1.6757787466049194, -1.0836637020111084 ], [ 2.0114352703094482, -2.3157713413238525, -0.17566370964050293 ], [ 2.3914389610290527, -0.8197798132896423, -2.655663013458252 ], ... ],
[ [ 1.5714348554611206, -1.9517707824707031, -2.5156569480895996 ], [ -0.8045579195022583, 0.3322182297706604, 1.1283330917358398 ], [ 0.1794452667236328, -1.339775800704956, -0.0036676526069641113 ], [ -1.3445556163787842, -1.0597765445709229, -1.5996662378311157 ], [ 1.3754384517669678, -0.3797750771045685, 0.16833490133285522 ], [ -0.172561377286911, 0.7522158026695251, 0.7443363666534424 ], [ 0.6354436278343201, -2.467773914337158, -0.32766443490982056 ], [ 2.0634379386901855, -2.455775260925293, -0.9396617412567139 ], ... ],
[ [ 2.16743803024292, -2.39577579498291, -1.479663372039795 ], [ 1.0314384698867798, -1.1437779664993286, 0.8963298797607422 ], [ 1.8394410610198975, -1.4357788562774658, -0.4116615951061249 ], [ -0.9085561037063599, -2.379777669906616, -1.4836618900299072 ], [ 0.7954368591308594, -1.2677782773971558, -0.9516671895980835 ], [ -0.33655866980552673, -0.34377530217170715, -2.7796599864959717 ], [ 0.8874431252479553, -0.10377514362335205, -1.4236624240875244 ], [ -0.540554404258728, -2.6997737884521484, -1.5156607627868652 ], ... ],
[ [ -0.684558629989624, -1.0117759704589844, -1.8916655778884888 ], [ 0.47144338488578796, -2.315770387649536, -0.9436672329902649 ], [ 1.3074376583099365, -1.531775951385498, 0.8203309774398804 ], [ 0.7994401454925537, -1.587774395942688, -1.6196624040603638 ], [ 1.4234422445297241, -1.8677754402160645, 0.10833334922790527 ], [ 2.0074377059936523, 0.5082185864448547, 0.4643300771713257 ], [ 0.8594357967376709, -0.12777525186538696, 0.9443336725234985 ], [ -0.6685571670532227, 0.7002199292182922, 0.32033073902130127 ], ... ],
...
]
TrainingTester.java:432 executed in 0.17 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[1.0, 0.3333892487517569], [5.0, 0.4753555933797641]; valueStats=DoubleSummaryStatistics{count=11, sum=29.567273, min=2.154712, average=2.687934, max=2.987828}
Plotting 5 points for GD
Plotting 2 points for CjGD
Plotting 4 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
Plotting range=[0.0, 0.3333892487517569], [41.924, 0.4753555933797641]; valueStats=DoubleSummaryStatistics{count=11, sum=29.567273, min=2.154712, average=2.687934, max=2.987828}
Plotting 5 points for GD
Plotting 2 points for CjGD
Plotting 4 points for LBFGS
Returns
TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):
return grid(inputLearning, modelLearning, completeLearning);
Returns
TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):
return new ComponentResult(null == inputLearning ? null : inputLearning.value,
null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);
Returns
{"input":{ "LBFGS": { "type": "NonConverged", "value": 2.666953379661158 }, "CjGD": { "type": "NonConverged", "value": 2.1547120913747952 }, "GD": { "type": "NonConverged", "value": 2.6665597003108523 } }, "model":null, "complete":null}
LayerTests.java:425 executed in 0.00 seconds (0.000 gc):
throwException(exceptions.addRef());
| details | result |
|---|---|
| {"input":{ "LBFGS": { "type": "NonConverged", "value": 2.666953379661158 }, "CjGD": { "type": "NonConverged", "value": 2.1547120913747952 }, "GD": { "type": "NonConverged", "value": 2.6665597003108523 } }, "model":null, "complete":null} | OK |
{
"result": "OK",
"performance": {
"execution_time": "130.694",
"gc_time": "7.163"
},
"created_on": 1586740004620,
"file_name": "trainingTest",
"report": {
"simpleName": "Float",
"canonicalName": "com.simiacryptus.mindseye.layers.cudnn.GateBiasLayerTest.Float",
"link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/test/java/com/simiacryptus/mindseye/layers/cudnn/GateBiasLayerTest.java",
"javaDoc": ""
},
"training_analysis": {
"input": {
"LBFGS": {
"type": "NonConverged",
"value": 2.666953379661158
},
"CjGD": {
"type": "NonConverged",
"value": 2.1547120913747952
},
"GD": {
"type": "NonConverged",
"value": 2.6665597003108523
}
}
},
"archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/cudnn/GateBiasLayer/Float/trainingTest/202004130644",
"id": "d0d17ece-73e6-4676-b4f8-a5aa1104b7f6",
"report_type": "Components",
"display_name": "Comparative Training",
"target": {
"simpleName": "GateBiasLayer",
"canonicalName": "com.simiacryptus.mindseye.layers.cudnn.GateBiasLayer",
"link": "https://github.com/SimiaCryptus/mindseye-cudnn/tree/59d5b3318556370acb2d83ee6ec123ce0fc6974f/src/main/java/com/simiacryptus/mindseye/layers/cudnn/GateBiasLayer.java",
"javaDoc": ""
}
}