Subreport: Logs for com.simiacryptus.ref.lang.ReferenceCountingBase
Using Seed 2250267197474510848
In this apply, we use a network to learn this target input, given it's pre-evaluated output:
TrainingTester.java:332 executed in 0.03 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(input_target)).flatMap(RefArrays::stream).map(x -> {
try {
return x.prettyPrint();
} finally {
x.freeRef();
}
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ -0.004894592941887321 ], [ -0.9759231480515884 ], [ 0.6949761656013611 ], [ 0.985593909003756 ], [ -0.5423619943271212 ], [ 0.5190097847498623 ], [ -0.4006801892837253 ], [ -0.743716036013564 ], ... ],
[ [ 0.9127316376504118 ], [ -0.7918180483013255 ], [ -0.7050635368788829 ], [ 0.6491935125673447 ], [ 0.5864294664911827 ], [ 0.6012868794548019 ], [ 0.49321583649667944 ], [ 0.28623060869662664 ], ... ],
[ [ -0.5144125895886573 ], [ -0.28914626062637105 ], [ -0.5560261225913687 ], [ -0.5439753623840844 ], [ -0.38800043141620977 ], [ 0.04774337320776345 ], [ -0.6207023590404157 ], [ 0.22889910913375378 ], ... ],
[ [ -0.7477229293171468 ], [ -0.5220717717344359 ], [ -0.8807078291593684 ], [ -0.7023877961426201 ], [ -0.6909424800760926 ], [ -0.4703623990552026 ], [ -0.3030350337507177 ], [ 0.0703918104900485 ], ... ],
[ [ -0.05863346342565756 ], [ 0.6020494156108536 ], [ 0.12968279379291325 ], [ -0.2633114121972888 ], [ -0.08157089312579147 ], [ -0.5748137984769204 ], [ -0.4653511899674856 ], [ -0.8086233302649204 ], ... ],
[ [ 0.5744252806174053 ], [ 0.8489706482917851 ], [ -0.6740592706828339 ], [ -0.057641078770721776 ], [ -0.25463279281870665 ], [ -0.40820334848529094 ], [ 0.7028876555242329 ], [ -0.2905242716768277 ], ... ],
[ [ 0.16770936663441893 ], [ -0.9401694724255452 ], [ -0.24136924473358534 ], [ 0.7704461263312834 ], [ 0.8610068015851529 ], [ 0.7851229228242897 ], [ 0.9917141102673168 ], [ 0.4185015655851254 ], ... ],
[ [ 0.837992924310573 ], [ -0.603271347446227 ], [ -0.7620141821581059 ], [ -0.19561791599649303 ], [ 0.08556010608386333 ], [ 0.6059589476059304 ], [ -0.2964425430408866 ], [ -0.24068178427081044 ], ... ],
...
]
[
[ [ 0.09444141526818872 ], [ 0.5637376759740858 ], [ -0.4821913299519771 ], [ 0.6224389396972683 ], [ -0.9914265627712335 ], [ 0.43029180796981237 ], [ -0.669591928729341 ], [ -0.7233531460674432 ], ... ],
[ [ -0.51890586605656 ], [ -0.4349894474211129 ], [ -0.29707072997806083 ], [ 0.04680345198104807 ], [ -0.39956605664639344 ], [ 0.5959860745755079 ], [ -0.30833961513220354 ], [ 0.7566909210009789 ], ... ],
[ [ -0.937438622736041 ], [ -0.8636071970457482 ], [ -0.4946207595839327 ], [ -0.12600420175546878 ], [ -0.9612060082456335 ], [ -0.31911669509084617 ], [ -0.4671348045686192 ], [ 0.7747728587335345 ], ... ],
[ [ 0.9563310578200324 ], [ -0.44534119739103173 ], [ 0.638007820761956 ], [ -0.24399729298574746 ], [ -0.7713467261873923 ], [ 0.4845039000220601 ], [ 0.3149383007574633 ], [ -0.5710230987713769 ], ... ],
[ [ -0.44099484887093765 ], [ -0.6389425053678714 ], [ -0.7926194239574269 ], [ 0.7435192093837815 ], [ -0.7032127180104295 ], [ 0.41997075812426465 ], [ 0.9877369102086866 ], [ -0.892471716787987 ], ... ],
[ [ 0.6102113446171198 ], [ -0.5022500658085829 ], [ 0.26884134169513363 ], [ 0.8047022498159444 ], [ 0.47527317676092085 ], [ -0.5169164339093684 ], [ -0.8320203996462672 ], [ 0.9158432892398927 ], ... ],
[ [ 0.3467483199557442 ], [ -0.042279342512006046 ], [ 0.20028127303234133 ], [ 0.40297405077969417 ], [ 0.6364011877140545 ], [ 0.48838817569636483 ], [ 0.5393453393088854 ], [ -0.5275414567533423 ], ... ],
[ [ -0.47019627032099875 ], [ -0.41240274316919767 ], [ 0.019508707815817594 ], [ -0.12629857456148197 ], [ -0.45415230776970694 ], [ 0.9679574950227423 ], [ -0.17720144013060346 ], [ 0.5764486431202729 ], ... ],
...
]
[
[ [ 0.42106033431921674 ], [ -0.9819345976223326 ], [ 0.5025867804255288 ], [ 0.5849253979217168 ], [ -0.6225196830434632 ], [ -0.8162270088523262 ], [ -0.7792682496311214 ], [ 0.18250018858867056 ], ... ],
[ [ 0.37183816827725624 ], [ 0.4723365975635734 ], [ -0.8038779850109601 ], [ -0.4964749290023749 ], [ -0.3546503489720292 ], [ -0.7120673717032893 ], [ 0.7760074605271527 ], [ 0.05735882638800138 ], ... ],
[ [ 0.6687164402318884 ], [ 0.22907621478668583 ], [ -0.3874952620695493 ], [ 0.8964209871560824 ], [ -0.3125244136780019 ], [ -0.21944439088293088 ], [ 0.06553372889994646 ], [ 0.10662625149789373 ], ... ],
[ [ 0.3100753094143984 ], [ 0.8964637798501045 ], [ -0.04290777525989253 ], [ -0.14092574054670903 ], [ 0.2580059049616894 ], [ 0.7218891605918156 ], [ -0.40823246107850486 ], [ -0.6537762889768686 ], ... ],
[ [ -0.8181383287149457 ], [ 0.6767510935371943 ], [ -0.35852608784642004 ], [ -0.8732706289504766 ], [ 0.09392320434772539 ], [ -0.45280389329506754 ], [ -0.4983944779099132 ], [ 0.7445527220061956 ], ... ],
[ [ -0.7508837162195041 ], [ 0.4479469857379752 ], [ 0.9649759605710877 ], [ -0.9101904813535002 ], [ 0.2189152996733046 ], [ -0.9130536826036773 ], [ 0.345890463056691 ], [ -0.85701343917082 ], ... ],
[ [ 0.5385013362078661 ], [ -0.2687155455796806 ], [ -0.808013013973951 ], [ 0.1542472762285998 ], [ 0.4304607119924315 ], [ 0.2646501583996257 ], [ -0.8583635739858062 ], [ -0.6925462747022126 ], ... ],
[ [ -0.08550688130907602 ], [ 0.3895230881754288 ], [ -0.04286641862656415 ], [ -0.8359218254873981 ], [ 0.206842582352317 ], [ -0.7946796185761602 ], [ -0.9820516514075688 ], [ -0.4763438106529094 ], ... ],
...
]
[
[ [ -0.16191588034940652 ], [ -0.22942347741832214 ], [ -0.47744506526061703 ], [ 0.8900518538611575 ], [ 0.2864861147915705 ], [ -0.9769108917665759 ], [ 0.5794755673976089 ], [ 0.21793582311740578 ], ... ],
[ [ 0.89092542632375 ], [ 0.8373162912210383 ], [ 0.557039101949778 ], [ -0.3433750909072788 ], [ 0.27666577054067143 ], [ 0.8023312706727908 ], [ 0.30958949086168497 ], [ -0.9710684898168467 ], ... ],
[ [ 0.7155054769557909 ], [ -0.666298214401656 ], [ 0.7165010053442131 ], [ -0.8876675969578727 ], [ -0.03983692052824783 ], [ -0.20899614938290356 ], [ 0.781699261146036 ], [ -0.04381945763601691 ], ... ],
[ [ 0.29616930768892025 ], [ -0.39967841468560317 ], [ -0.14593027225190958 ], [ 0.08827745271150356 ], [ -0.4955679089590561 ], [ -0.6381182254198186 ], [ -0.7964093472694644 ], [ 0.5703755938980724 ], ... ],
[ [ -0.0039990882902687375 ], [ 0.14524166164279229 ], [ -0.5670326955423708 ], [ 0.6006616773838491 ], [ 0.7072174199151318 ], [ -0.009108527632514019 ], [ 0.8821112878114057 ], [ -0.18046422180790156 ], ... ],
[ [ -0.8101949473741212 ], [ -0.07732671154307225 ], [ -0.5725904895624334 ], [ -0.39343784036254426 ], [ 0.8311174252405202 ], [ -0.5725539373363082 ], [ -0.8197113761951147 ], [ -0.48946733938456255 ], ... ],
[ [ -0.5416158458637104 ], [ 0.49614431068054543 ], [ -0.1749492651940323 ], [ 0.7166495863092548 ], [ -0.8778663167188676 ], [ -0.23985540779070869 ], [ 0.948228774281979 ], [ -0.24679567376593536 ], ... ],
[ [ -0.21434616828180286 ], [ -0.10308544640716999 ], [ 0.08372267323747296 ], [ 0.33927590395549867 ], [ 0.6768733739267767 ], [ -0.5027733080139134 ], [ -0.20564972757357003 ], [ -0.7418028576482694 ], ... ],
...
]
[
[ [ -0.53815281951085 ], [ 0.25928465245417764 ], [ -0.5755844566281689 ], [ -0.4194638592999924 ], [ 0.18462449372753786 ], [ -0.7154799510683423 ], [ 0.354382216255017 ], [ -0.9926820843494704 ], ... ],
[ [ 0.15550200498165134 ], [ -0.19774427720774068 ], [ -0.1667226042845953 ], [ 0.6849465238320556 ], [ -0.7448400751105266 ], [ 0.16121886405612273 ], [ 0.6125715054794032 ], [ -0.06197845666683244 ], ... ],
[ [ -0.5791760519342322 ], [ 0.46944508423245646 ], [ 0.18632019322092797 ], [ 0.4131001443768312 ], [ 0.3525882793815698 ], [ 0.2137230413506641 ], [ 0.44463571266587576 ], [ -0.880725036315972 ], ... ],
[ [ -0.9328317662666443 ], [ 0.9803628667420166 ], [ -0.45285706347524446 ], [ 0.6561615324973666 ], [ 0.5542872308908935 ], [ 0.09969344882945053 ], [ -0.763477120617654 ], [ 0.7239021178400407 ], ... ],
[ [ -0.040607885405664845 ], [ 0.0730370415310968 ], [ -0.0099457921398729 ], [ -0.7424340840480612 ], [ -0.8277732962658213 ], [ 0.6387075913313427 ], [ 0.5249778460079453 ], [ 0.8621019006731911 ], ... ],
[ [ -0.7436488625162245 ], [ 0.3511528130808659 ], [ 0.8276049945200876 ], [ -0.7132675117713714 ], [ 0.9069191358888309 ], [ 0.6796414573792668 ], [ 0.6107039670091037 ], [ -0.446458835953353 ], ... ],
[ [ -0.9641369111863334 ], [ 0.6574279961961766 ], [ -0.6781512952992503 ], [ -0.7130554825914537 ], [ -0.7345821823979608 ], [ -0.9701855613667105 ], [ 0.07974209888778727 ], [ 0.7669151267918204 ], ... ],
[ [ -0.6178197945369908 ], [ 0.7362889174224836 ], [ -0.25705117450345916 ], [ -0.7649097349458072 ], [ 0.5706664298635562 ], [ 0.22738872493616813 ], [ 0.3144108903169516 ], [ -0.8080597013798825 ], ... ],
...
]
First, we train using basic gradient descent method apply weak line search conditions.
TrainingTester.java:480 executed in 30.20 seconds (0.164 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 2769845243713
BACKPROP_AGG_SIZE = 3
THREADS = 64
SINGLE_THREADED = false
Initialized CoreSettings = {
"backpropAggregationSize" : 3,
"jvmThreads" : 64,
"singleThreaded" : false
}
Reset training subject: 2769930155462
Constructing line search parameters: GD
th(0)=15.162946939207576;dx=-5.645340560031366E9
New Minimum: 15.162946939207576 > 8.870446379707042
Armijo: th(2.154434690031884)=8.870446379707042; dx=-0.06121239539696214 evalInputDelta=6.292500559500533
Armijo: th(1.077217345015942)=8.950114943395853; dx=-0.12530897402939326 evalInputDelta=6.212831995811722
Armijo: th(0.3590724483386473)=9.064252424443843; dx=-0.40437638976051327 evalInputDelta=6.098694514763732
Armijo: th(0.08976811208466183)=9.262985820599429; dx=-1.6120565185253828 evalInputDelta=5.899961118608147
Armijo: th(0.017953622416932366)=9.458966290987814; dx=-10.1448197287502 evalInputDelta=5.703980648219762
Armijo: th(0.002992270402822061)=9.745679166317343; dx=-50.77172788049012 evalInputDelta=5.4172677728902325
END: th(4.2746720040315154E-4)=10.033227448182458; dx=-338.3238798728223 evalInputDelta=5.129719491025117
Fitness changed from 15.162946939207576 to 8.870446379707042
Iteration 1 complete. Error: 8.870446379707042 Total: 0.4063; Orientation: 0.0082; Line Search: 0.2774
th(0)=8.870446379707042;dx=-0.026583894433485716
New Minimum: 8.870446379707042 > 8.870421894620833
WOLFE (weak): th(9.20950165399361E-4)=8.870421894620833; dx=-0.026589640220459537 evalInputDelta=2.448508620922496E-5
New Minimum: 8.870421894620833 > 8.870397404233682
WOLFE (weak): th(0.001841900330798722)=8.870397404233682; dx=-0.026595406400326545 evalInputDelta=4.897547336035757E-5
New Minimum: 8.870397404233682 > 8.870299389297763
WOLFE (weak): th(0.005525700992396166)=8.870299389297763; dx=-0.02661867672048679 evalInputDelta=1.4699040927901308E-4
New Minimum: 8.870299389297763 > 8.869857235645897
WOLFE (weak): th(0.022102803969584663)=8.869857235645897; dx=-0.026727595252618357 evalInputDelta=5.891440611449639E-4
New Minimum: 8.869857235645897 > 8.867464310859116
WOLFE (weak): th(0.11051401984792332)=8.867464310859116; dx=-0.027446845262080466 evalInputDelta=0.002982068847925845
New Minimum: 8.867464310859116 > 8.84518362635804
WOLFE (weak): th(0.66308411908754)=8.84518362635804; dx=-0.10172704419204949 evalInputDelta=0.02526275334900241
New Minimum: 8.84518362635804 > 8.798367489241965
END: th(4.641588833612779)=8.798367489241965; dx=-0.015125659667823465 evalInputDelta=0.0720788904650771
Fitness changed from 8.870446379707042 to 8.798367489241965
Iteration 2 complete. Error: 8.798367489241965 Total: 0.3737; Orientation: 0.0045; Line Search: 0.3418
th(0)=8.798367489241965;dx=-0.037098459492995305
New Minimum: 8.798367489241965 > 8.736082573295942
END: th(10.000000000000002)=8.736082573295942; dx=-0.00848292359225599 evalInputDelta=0.062284915946023034
Fitness changed from 8.798367489241965 to 8.736082573295942
Iteration 3 complete. Error: 8.736082573295942 Total: 0.0521; Orientation: 0.0042; Line Search: 0.0350
th(0)=8.736082573295942;dx=-0.027612967850453043
New Minimum: 8.736082573295942 > 8.694750700959869
WOLF (strong): th(21.544346900318843)=8.694750700959869; dx=0.014607240413491323 evalInputDelta=0.04133187233607316
END: th(10.772173450159421)=8.71691722007007; dx=-0.00502867532559985 evalInputDelta=0.01916535322587265
Fitness changed from 8.736082573295942 to 8.694750700959869
Iteration 4 complete. Error: 8.694750700959869 Total: 0.0657; Orientation: 0.0034; Line Search: 0.0491
th(0)=8.694750700959869;dx=-24.102947571260714
New Minimum: 8.694750700959869 > 8.647291390398157
END: th(23.2079441680639)=8.647291390398157; dx=-0.003233290185055009 evalInputDelta=0.047459310561711376
Fitness changed from 8.694750700959869 to 8.647291390398157
Iteration 5 complete. Error: 8.647291390398157 Total: 0.1459; Orientation: 0.0030; Line Search: 0.1320
th(0)=8.647291390398157;dx=-0.016251577890454522
New Minimum: 8.647291390398157 > 8.623749276896175
WOLF (strong): th(50.000000000000014)=8.623749276896175; dx=0.0027909892099835694 evalInputDelta=0.023542113501981987
END: th(25.000000000000007)=8.640935805064176; dx=-0.0020292519877715062 evalInputDelta=0.006355585333981395
Fitness changed from 8.647291390398157 to 8.623749276896175
Iteration 6 complete. Error: 8.623749276896175 Total: 0.1101; Orientation: 0.0036; Line Search: 0.0532
th(0)=8.623749276896175;dx=-24.844887429604494
New Minimum: 8.623749276896175 > 8.618510396045021
WOLF (strong): th(53.860867250797114)=8.618510396045021; dx=0.012477542561676382 evalInputDelta=0.005238880851154093
New Minimum: 8.618510396045021 > 8.614251903275152
WOLF (strong): th(26.930433625398557)=8.614251903275152; dx=0.10458055297515853 evalInputDelta=0.009497373621023542
WOLF (strong): th(8.976811208466186)=8.62053952006671; dx=0.0051830977997543125 evalInputDelta=0.0032097568294648937
WOLF (strong): th(2.2442028021165465)=8.62369003154869; dx=0.07494255636096012 evalInputDelta=5.9245347484804256E-5
Armijo: th(0.44884056042330933)=8.637023134368054; dx=1.6508959209367882 evalInputDelta=-0.013273857471878259
WOLF (strong): th(0.07480676007055155)=8.62016779600906; dx=0.4301080451382193 evalInputDelta=0.003581480887115518
END: th(0.010686680010078792)=8.616662895532532; dx=-0.006435924980491674 evalInputDelta=0.007086381363643923
Fitness changed from 8.623749276896175 to 8.614251903275152
Iteration 7 complete. Error: 8.614251903275152 Total: 0.7179; Orientation: 0.0039; Line Search: 0.7020
th(0)=8.614251903275152;dx=-567.1998458672236
New Minimum: 8.614251903275152 > 8.603477660632128
END: th(0.02302375413498403)=8.603477660632128; dx=-0.056167946526488814 evalInputDelta=0.010774242643023513
Fitness changed from 8.614251903275152 to 8.603477660632128
Iteration 8 complete. Error: 8.603477660632128 Total: 0.0693; Orientation: 0.0039; Line Search: 0.0543
th(0)=8.603477660632128;dx=-0.08211111837308034
New Minimum: 8.603477660632128 > 8.595637898281234
WOLFE (weak): th(0.04960317460317463)=8.595637898281234; dx=-0.33445398643120977 evalInputDelta=0.00783976235089412
Armijo: th(0.09920634920634926)=8.615213163961503; dx=-0.004456139709027377 evalInputDelta=-0.011735503329374453
New Minimum: 8.595637898281234 > 8.57329866473248
WOLF (strong): th(0.07440476190476195)=8.57329866473248; dx=15.40481751280749 evalInputDelta=0.030178995899648342
WOLFE (weak): th(0.062003968253968283)=8.589416302192713; dx=-0.7707148485084896 evalInputDelta=0.014061358439414917
WOLFE (weak): th(0.06820436507936511)=8.582634227677138; dx=-1.572766349834066 evalInputDelta=0.02084343295499025
WOLFE (weak): th(0.07130456349206353)=8.57634947096974; dx=-2.5876933525374315 evalInputDelta=0.02712818966238828
New Minimum: 8.57329866473248 > 8.571943757503789
WOLFE (weak): th(0.07285466269841273)=8.571943757503789; dx=-2.8992994748850016 evalInputDelta=0.0315339031283397
New Minimum: 8.571943757503789 > 8.570135603788254
WOLFE (weak): th(0.07362971230158734)=8.570135603788254; dx=-1.1566850888929892 evalInputDelta=0.0333420568438747
WOLF (strong): th(0.07401723710317465)=8.570302215134651; dx=2.7088709955531756 evalInputDelta=0.033175445497477085
mu ~= nu (0.07362971230158734): th(0.07362971230158734)=8.570135603788254
Fitness changed from 8.603477660632128 to 8.570135603788254
Iteration 9 complete. Error: 8.570135603788254 Total: 1.0257; Orientation: 0.0028; Line Search: 1.0111
th(0)=8.570135603788254;dx=-17.695407565923063
Armijo: th(0.1590478548375008)=8.61472999335366; dx=-0.003685020454969912 evalInputDelta=-0.044594389565405734
Armijo: th(0.0795239274187504)=8.615026122858783; dx=-0.0037718248522997373 evalInputDelta=-0.04489051907052932
Armijo: th(0.026507975806250136)=8.615228433430236; dx=-0.0038660001235549767 evalInputDelta=-0.045092829641982135
Armijo: th(0.006626993951562534)=8.61530573109091; dx=-0.0039110300804269995 evalInputDelta=-0.04517012730265613
Armijo: th(0.0013253987903125068)=8.61532650015709; dx=-0.003924070927101638 evalInputDelta=-0.045190896368836775
Armijo: th(2.208997983854178E-4)=8.615330835819798; dx=-0.003926845346315963 evalInputDelta=-0.04519523203154385
Armijo: th(3.1557114055059684E-5)=8.570653978888014; dx=70.38497453288501 evalIn
...skipping 38018 bytes...
25;dx=-0.055127845068254855
Armijo: th(4.156396562104444E-5)=8.570019322058503; dx=0.1854359752921141 evalInputDelta=-2.787775777335355E-6
Armijo: th(2.078198281052222E-5)=8.570016678577607; dx=0.06803072984865442 evalInputDelta=-1.4429488182088335E-7
New Minimum: 8.570016534282725 > 8.570016297250312
END: th(6.9273276035074075E-6)=8.570016297250312; dx=-0.013417349112817044 evalInputDelta=2.3703241325279123E-7
Fitness changed from 8.570016534282725 to 8.570016297250312
Iteration 83 complete. Error: 8.570016297250312 Total: 0.4138; Orientation: 0.0458; Line Search: 0.2843
th(0)=8.570016297250312;dx=-0.005671994396752817
New Minimum: 8.570016297250312 > 8.570016235488534
END: th(1.4924474898211794E-5)=8.570016235488534; dx=-0.0026078480320487676 evalInputDelta=6.176177791417103E-8
Fitness changed from 8.570016297250312 to 8.570016235488534
Iteration 84 complete. Error: 8.570016235488534 Total: 0.1831; Orientation: 0.0030; Line Search: 0.1703
th(0)=8.570016235488534;dx=-0.0048594175782904914
New Minimum: 8.570016235488534 > 8.570016136559197
END: th(3.2153806451217555E-5)=8.570016136559197; dx=-0.0012880643890837897 evalInputDelta=9.892933761079803E-8
Fitness changed from 8.570016235488534 to 8.570016136559197
Iteration 85 complete. Error: 8.570016136559197 Total: 0.1972; Orientation: 0.0030; Line Search: 0.1618
th(0)=8.570016136559197;dx=-0.011089592545355784
Armijo: th(6.927327603507408E-5)=8.570017346326733; dx=0.04545398195195585 evalInputDelta=-1.209767535925721E-6
Armijo: th(3.463663801753704E-5)=8.570016251862151; dx=0.01760444075301149 evalInputDelta=-1.153029547396045E-7
New Minimum: 8.570016136559197 > 8.570016064382589
END: th(1.154554600584568E-5)=8.570016064382589; dx=-0.001429403079967908 evalInputDelta=7.21766078015662E-8
Fitness changed from 8.570016136559197 to 8.570016064382589
Iteration 86 complete. Error: 8.570016064382589 Total: 0.4185; Orientation: 0.0964; Line Search: 0.3109
th(0)=8.570016064382589;dx=-0.004758929108113537
New Minimum: 8.570016064382589 > 8.570015976681223
END: th(2.4874124830352993E-5)=8.570015976681223; dx=-0.002289637163620797 evalInputDelta=8.770136616931268E-8
Fitness changed from 8.570016064382589 to 8.570015976681223
Iteration 87 complete. Error: 8.570015976681223 Total: 0.1715; Orientation: 0.0027; Line Search: 0.0977
th(0)=8.570015976681223;dx=-0.006964523783577398
Armijo: th(5.358967741869594E-5)=8.570016111174471; dx=0.011889042150080313 evalInputDelta=-1.344932485380923E-7
New Minimum: 8.570015976681223 > 8.570015917635835
END: th(2.679483870934797E-5)=8.570015917635835; dx=0.002533398446542711 evalInputDelta=5.9045387246214887E-8
Fitness changed from 8.570015976681223 to 8.570015917635835
Iteration 88 complete. Error: 8.570015917635835 Total: 0.1389; Orientation: 0.0204; Line Search: 0.1093
th(0)=8.570015917635835;dx=-0.01745705771398817
Armijo: th(5.772773002922841E-5)=8.570017566964363; dx=0.07566357592677 evalInputDelta=-1.649328527975058E-6
Armijo: th(2.8863865014614206E-5)=8.570016070415702; dx=0.028304619667597158 evalInputDelta=-1.5277986697981305E-7
New Minimum: 8.570015917635835 > 8.570015822083157
END: th(9.621288338204736E-6)=8.570015822083157; dx=-0.0023770249069526915 evalInputDelta=9.555267865835049E-8
Fitness changed from 8.570015917635835 to 8.570015822083157
Iteration 89 complete. Error: 8.570015822083157 Total: 0.2998; Orientation: 0.0027; Line Search: 0.1718
th(0)=8.570015822083157;dx=-0.004077536873060226
New Minimum: 8.570015822083157 > 8.570015741865125
WOLFE (weak): th(2.07284373586275E-5)=8.570015741865125; dx=-0.0036625571645478505 evalInputDelta=8.021803132862715E-8
New Minimum: 8.570015741865125 > 8.57001567023705
END: th(4.1456874717255E-5)=8.57001567023705; dx=-0.0032487264965244604 evalInputDelta=1.5184610724361391E-7
Fitness changed from 8.570015822083157 to 8.57001567023705
Iteration 90 complete. Error: 8.57001567023705 Total: 0.3101; Orientation: 0.0031; Line Search: 0.1692
th(0)=8.57001567023705;dx=-0.006405064921343391
Armijo: th(8.931612903115991E-5)=8.570016269381686; dx=0.0200220822098526 evalInputDelta=-5.991446361264252E-7
Armijo: th(4.4658064515579956E-5)=8.570015674776798; dx=0.006657949741917707 evalInputDelta=-4.5397481329700895E-9
New Minimum: 8.57001567023705 > 8.570015607014195
END: th(1.4886021505193318E-5)=8.570015607014195; dx=-0.002083730604339822 evalInputDelta=6.322285450721665E-8
Fitness changed from 8.57001567023705 to 8.570015607014195
Iteration 91 complete. Error: 8.570015607014195 Total: 0.1809; Orientation: 0.0029; Line Search: 0.1677
th(0)=8.570015607014195;dx=-0.005231692194534934
New Minimum: 8.570015607014195 > 8.570015518442496
END: th(3.2070961127349125E-5)=8.570015518442496; dx=-3.015712307167848E-4 evalInputDelta=8.857169930820419E-8
Fitness changed from 8.570015607014195 to 8.570015518442496
Iteration 92 complete. Error: 8.570015518442496 Total: 0.0810; Orientation: 0.0030; Line Search: 0.0682
th(0)=8.570015518442496;dx=-0.013690936535138514
Armijo: th(6.909479119542501E-5)=8.570017326615341; dx=0.06696815201753181 evalInputDelta=-1.8081728452301604E-6
Armijo: th(3.4547395597712506E-5)=8.570015725964703; dx=0.02593489421526722 evalInputDelta=-2.0752220741826477E-7
New Minimum: 8.570015518442496 > 8.57001543580708
END: th(1.1515798532570836E-5)=8.57001543580708; dx=-6.354657597254644E-4 evalInputDelta=8.263541495523441E-8
Fitness changed from 8.570015518442496 to 8.57001543580708
Iteration 93 complete. Error: 8.57001543580708 Total: 0.4357; Orientation: 0.0092; Line Search: 0.3975
th(0)=8.57001543580708;dx=-0.004996931552720116
New Minimum: 8.57001543580708 > 8.570015350892554
END: th(2.481003584198887E-5)=8.570015350892554; dx=-0.0018526231287475617 evalInputDelta=8.491452696546276E-8
Fitness changed from 8.57001543580708 to 8.570015350892554
Iteration 94 complete. Error: 8.570015350892554 Total: 0.1093; Orientation: 0.0033; Line Search: 0.0969
th(0)=8.570015350892554;dx=-0.007760133857789964
Armijo: th(5.345160187891522E-5)=8.570015579770516; dx=0.016460734450539843 evalInputDelta=-2.288779619163961E-7
New Minimum: 8.570015350892554 > 8.570015303505164
END: th(2.672580093945761E-5)=8.570015303505164; dx=0.004247800320825952 evalInputDelta=4.7387390011977004E-8
Fitness changed from 8.570015350892554 to 8.570015303505164
Iteration 95 complete. Error: 8.570015303505164 Total: 0.2954; Orientation: 0.0034; Line Search: 0.2828
th(0)=8.570015303505164;dx=-0.02114244590567925
Armijo: th(5.757899266285418E-5)=8.570017359780179; dx=0.09112813477705624 evalInputDelta=-2.056275015149822E-6
Armijo: th(2.878949633142709E-5)=8.570015523695924; dx=0.036071907806053544 evalInputDelta=-2.2019075984758274E-7
New Minimum: 8.570015303505164 > 8.570015193496829
END: th(9.596498777142363E-6)=8.570015193496829; dx=-0.0018256720626375687 evalInputDelta=1.1000833488594708E-7
Fitness changed from 8.570015303505164 to 8.570015193496829
Iteration 96 complete. Error: 8.570015193496829 Total: 0.3732; Orientation: 0.0045; Line Search: 0.3149
th(0)=8.570015193496829;dx=-0.004155150461071813
New Minimum: 8.570015193496829 > 8.570015113812948
WOLFE (weak): th(2.067502986832406E-5)=8.570015113812948; dx=-0.0035527405990730555 evalInputDelta=7.968388082701949E-8
New Minimum: 8.570015113812948 > 8.570015046604677
END: th(4.135005973664812E-5)=8.570015046604677; dx=-0.0029483173701446005 evalInputDelta=1.4689215177554615E-7
Fitness changed from 8.570015193496829 to 8.570015046604677
Iteration 97 complete. Error: 8.570015046604677 Total: 0.2816; Orientation: 0.0027; Line Search: 0.2700
th(0)=8.570015046604677;dx=-0.007550230955245849
Armijo: th(8.908600313152537E-5)=8.570016033381735; dx=0.029366816966734228 evalInputDelta=-9.867770582872026E-7
Armijo: th(4.454300156576269E-5)=8.570015128921682; dx=0.011160867628255104 evalInputDelta=-8.231700476812875E-8
New Minimum: 8.570015046604677 > 8.570014981298746
END: th(1.4847667188587562E-5)=8.570014981298746; dx=-0.00125615057171585 evalInputDelta=6.530593132936247E-8
Fitness changed from 8.570015046604677 to 8.570014981298746
Iteration 98 complete. Error: 8.570014981298746 Total: 0.3496; Orientation: 0.0023; Line Search: 0.2573
Final threshold in iteration 98: 8.570014981298746 (> 0.0) after 30.188s (< 30.000s)
Returns
8.570014981298746
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.36980055404510265 ], [ 0.7787287142004329 ], [ 0.5012602578153301 ], [ -0.567604295484103 ], [ -0.24575576717907066 ], [ -0.7352731424615937 ], [ -0.07668281805021615 ], [ 0.8371540726314336 ], ... ],
[ [ 0.5424263513546643 ], [ -0.11958585309344971 ], [ 0.27189287991969036 ], [ 0.3221543632910264 ], [ -0.8739393093920977 ], [ -0.1979201331861813 ], [ -0.7174800334829694 ], [ 0.15498281747640033 ], ... ],
[ [ -0.6679796127011541 ], [ 0.2859865857551899 ], [ 0.6853464797149263 ], [ 0.3991020949063914 ], [ 0.9256675904131623 ], [ 0.7027240811676257 ], [ -0.9075258636514625 ], [ -0.46057625089960097 ], ... ],
[ [ -0.23215044345743707 ], [ -0.3890998653356017 ], [ -0.9785746064056067 ], [ 0.24423849020213356 ], [ 0.5620518121024903 ], [ 0.761320706427388 ], [ 0.30501682877415337 ], [ 0.975367487803478 ], ... ],
[ [ 0.3306583097060603 ], [ 0.19641943192174577 ], [ -0.4611774915116107 ], [ 0.31007138549173213 ], [ -0.26645366817651706 ], [ -0.9233934703725706 ], [ -0.6031859486728848 ], [ 0.7492381420614942 ], ... ],
[ [ 0.3664396074112614 ], [ -0.4153968808034918 ], [ -0.9742419893501422 ], [ 0.21873885109525398 ], [ -0.8716834275485836 ], [ -0.9817172049762419 ], [ 0.9851495200552949 ], [ 0.2987651963616473 ], ... ],
[ [ 0.27285038966880293 ], [ -0.7664397461318597 ], [ -0.6227997209767129 ], [ -0.5122768236246127 ], [ 0.2037408941347088 ], [ 0.3170755893594419 ], [ 0.6655924032538569 ], [ 0.30462552742531046 ], ... ],
[ [ -0.49057723858683744 ], [ 0.9735651405214637 ], [ 0.36220649805580324 ], [ -0.48238711527764155 ], [ 0.10252436729063197 ], [ 0.6460055822751666 ], [ 0.23405240415247203 ], [ 0.42363473929013007 ], ... ],
...
]
To produce the following output:
TrainingTester.java:633 executed in 0.01 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ 1.6444331429841448 ], [ 1.1332008865001912 ], [ 1.41243465080393 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 1.0929424713639508 ], ... ],
[ [ 1.357780634437001 ], [ 0.0 ], [ 1.9177901450311488 ], [ 1.7618461877941576 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 2.540143337689325 ], ... ],
[ [ 0.0 ], [ 1.869937833335112 ], [ 1.207938731655171 ], [ 1.5829164641812836 ], [ 1.0393754887600766 ], [ 1.1929097324777251 ], [ 0.0 ], [ 0.0 ], ... ],
[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 2.0234521868007422 ], [ 1.3338648366285313 ], [ 1.1460832861007961 ], [ 1.8106649686347753 ], [ 1.0125485649336083 ], ... ],
[ [ 1.739042834711953 ], [ 2.2563567922155388 ], [ 0.0 ], [ 1.7958462622207065 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 1.155287464085639 ], ... ],
[ [ 1.6519572162295295 ], [ 0.0 ], [ 0.0 ], [ 2.138144422778811 ], [ 0.0 ], [ 0.0 ], [ 1.0075089781960094 ], [ 1.8295108851853736 ], ... ],
[ [ 1.9144221516351627 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 2.215444608660017 ], [ 1.7759003622261518 ], [ 1.2257328395113292 ], [ 1.8118275242956767 ], ... ],
[ [ 0.0 ], [ 1.013485389921675 ], [ 1.6615823921311517 ], [ 0.0 ], [ 3.1231040325951813 ], [ 1.2441761288500826 ], [ 2.0670141361874963 ], [ 1.5363997111434804 ], ... ],
...
]
First, we use a conjugate gradient descent method, which converges the fastest for purely linear functions.
TrainingTester.java:452 executed in 30.28 seconds (0.103 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new QuadraticSearch());
iterativeTrainer.setOrientation(new GradientDescent());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 2800059154464
Reset training subject: 2800071130319
Constructing line search parameters: GD
F(0.0) = LineSearchPoint{point=PointSample{avg=15.162946939207576}, derivative=-5.645340560031366E9}
New Minimum: 15.162946939207576 > 14.700488091554572
F(1.0E-10) = LineSearchPoint{point=PointSample{avg=14.700488091554572}, derivative=-3.820175704180969E9}, evalInputDelta = -0.4624588476530036
New Minimum: 14.700488091554572 > 13.48417807550735
F(7.000000000000001E-10) = LineSearchPoint{point=PointSample{avg=13.48417807550735}, derivative=-1.1449698739612684E9}, evalInputDelta = -1.6787688637002258
New Minimum: 13.48417807550735 > 12.110846802067295
F(4.900000000000001E-9) = LineSearchPoint{point=PointSample{avg=12.110846802067295}, derivative=-1.0443697067497775E8}, evalInputDelta = -3.0521001371402807
New Minimum: 12.110846802067295 > 11.460798802208188
F(3.430000000000001E-8) = LineSearchPoint{point=PointSample{avg=11.460798802208188}, derivative=-6444076.804793017}, evalInputDelta = -3.702148136999387
New Minimum: 11.460798802208188 > 11.092808870824395
F(2.4010000000000004E-7) = LineSearchPoint{point=PointSample{avg=11.092808870824395}, derivative=-674837.4383081932}, evalInputDelta = -4.07013806838318
New Minimum: 11.092808870824395 > 10.824429829783224
F(1.6807000000000003E-6) = LineSearchPoint{point=PointSample{avg=10.824429829783224}, derivative=-74803.88547900203}, evalInputDelta = -4.338517109424352
New Minimum: 10.824429829783224 > 10.56229970683915
F(1.1764900000000001E-5) = LineSearchPoint{point=PointSample{avg=10.56229970683915}, derivative=-12426.471211087486}, evalInputDelta = -4.600647232368425
New Minimum: 10.56229970683915 > 10.271756125250894
F(8.235430000000001E-5) = LineSearchPoint{point=PointSample{avg=10.271756125250894}, derivative=-1788.2801157106323}, evalInputDelta = -4.8911908139566815
New Minimum: 10.271756125250894 > 9.98986909701678
F(5.764801000000001E-4) = LineSearchPoint{point=PointSample{avg=9.98986909701678}, derivative=-252.16785776759897}, evalInputDelta = -5.173077842190796
New Minimum: 9.98986909701678 > 9.699988756513017
F(0.004035360700000001) = LineSearchPoint{point=PointSample{avg=9.699988756513017}, derivative=-38.07987200712995}, evalInputDelta = -5.462958182694559
New Minimum: 9.699988756513017 > 9.314970628113524
F(0.028247524900000005) = LineSearchPoint{point=PointSample{avg=9.314970628113524}, derivative=-50.925984008001784}, evalInputDelta = -5.847976311094051
New Minimum: 9.314970628113524 > 9.150001560773662
F(0.19773267430000002) = LineSearchPoint{point=PointSample{avg=9.150001560773662}, derivative=-0.719559134799165}, evalInputDelta = -6.012945378433914
New Minimum: 9.150001560773662 > 8.9276337534459
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=8.9276337534459}, derivative=0.5777329853493386}, evalInputDelta = -6.235313185761676
8.9276337534459 <= 15.162946939207576
Converged to right
Fitness changed from 15.162946939207576 to 8.9276337534459
Iteration 1 complete. Error: 8.9276337534459 Total: 0.9698; Orientation: 0.0026; Line Search: 0.9336
F(0.0) = LineSearchPoint{point=PointSample{avg=8.9276337534459}, derivative=-643.3865280449227}
New Minimum: 8.9276337534459 > 8.875360802028833
F(1.3841287201) = LineSearchPoint{point=PointSample{avg=8.875360802028833}, derivative=-0.0541823267940023}, evalInputDelta = -0.05227295141706634
New Minimum: 8.875360802028833 > 8.750606252342163
F(9.688901040700001) = LineSearchPoint{point=PointSample{avg=8.750606252342163}, derivative=-0.010997225989164812}, evalInputDelta = -0.177027501103737
New Minimum: 8.750606252342163 > 8.560237032350363
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=8.560237032350363}, derivative=6.142763467775788E-4}, evalInputDelta = -0.36739672109553645
8.560237032350363 <= 8.9276337534459
Converged to right
Fitness changed from 8.9276337534459 to 8.560237032350363
Iteration 2 complete. Error: 8.560237032350363 Total: 0.4560; Orientation: 0.0031; Line Search: 0.4429
F(0.0) = LineSearchPoint{point=PointSample{avg=8.560237032350363}, derivative=-4.522332144150844}
F(67.8223072849) = LineSearchPoint{point=PointSample{avg=8.562145216891324}, derivative=0.014294847363488586}, evalInputDelta = 0.0019081845409605336
New Minimum: 8.560237032350363 > 8.553618276310605
F(5.217100560376924) = LineSearchPoint{point=PointSample{avg=8.553618276310605}, derivative=0.0060289498605596865}, evalInputDelta = -0.006618756039758367
8.553618276310605 <= 8.560237032350363
Converged to right
Fitness changed from 8.560237032350363 to 8.553618276310605
Iteration 3 complete. Error: 8.553618276310605 Total: 0.1192; Orientation: 0.0029; Line Search: 0.1012
F(0.0) = LineSearchPoint{point=PointSample{avg=8.553618276310605}, derivative=-0.8966835437296188}
F(5.217100560376924) = LineSearchPoint{point=PointSample{avg=8.558738962214068}, derivative=-8.165684925914101E-4}, evalInputDelta = 0.005120685903463595
New Minimum: 8.553618276310605 > 8.548843951034414
F(0.40131542772130185) = LineSearchPoint{point=PointSample{avg=8.548843951034414}, derivative=-0.018050598799132088}, evalInputDelta = -0.004774325276191149
F(2.809207994049113) = LineSearchPoint{point=PointSample{avg=8.55749833733886}, derivative=-0.003044962472766267}, evalInputDelta = 0.0038800610282549997
F(0.21609292261916255) = LineSearchPoint{point=PointSample{avg=8.551129257400794}, derivative=-0.007944850952880472}, evalInputDelta = -0.002489018909811236
F(1.5126504583341378) = LineSearchPoint{point=PointSample{avg=8.560575552912226}, derivative=-0.0018951358765194062}, evalInputDelta = 0.006957276601621487
F(0.11635772756416445) = LineSearchPoint{point=PointSample{avg=8.55171147607038}, derivative=-0.0035273294693875185}, evalInputDelta = -0.0019068002402242712
F(0.8145040929491512) = LineSearchPoint{point=PointSample{avg=8.556740961603555}, derivative=-0.0047495782999458815}, evalInputDelta = 0.0031226852929506776
F(0.06265416099608855) = LineSearchPoint{point=PointSample{avg=8.551813740030974}, derivative=-6.519188107695156E-5}, evalInputDelta = -0.0018045362796303976
New Minimum: 8.548843951034414 > 8.548105510179909
F(0.4385791269726198) = LineSearchPoint{point=PointSample{avg=8.548105510179909}, derivative=-0.021768083710115828}, evalInputDelta = -0.005512766130696178
F(3.0700538888083386) = LineSearchPoint{point=PointSample{avg=8.556640127480563}, derivative=-0.0035748558183228726}, evalInputDelta = 0.003021851169958012
F(0.23615799144679528) = LineSearchPoint{point=PointSample{avg=8.550961641700633}, derivative=-0.00876448614689982}, evalInputDelta = -0.0026566346099716753
F(1.653105940127567) = LineSearchPoint{point=PointSample{avg=8.560303177155483}, derivative=-0.001983209550322285}, evalInputDelta = 0.006684900844877717
F(0.12716199539442824) = LineSearchPoint{point=PointSample{avg=8.551670315379928}, derivative=-0.0040862846780744}, evalInputDelta = -0.0019479609306767287
F(0.8901339677609976) = LineSearchPoint{point=PointSample{avg=8.55697152100169}, derivative=0.024402250610930924}, evalInputDelta = 0.003353244691085777
F(0.0684718436739229) = LineSearchPoint{point=PointSample{avg=8.551812077866604}, derivative=-5.040665644706951E-4}, evalInputDelta = -0.0018061984440009837
New Minimum: 8.548105510179909 > 8.547109566709661
F(0.47930290571746026) = LineSearchPoint{point=PointSample{avg=8.547109566709661}, derivative=-0.027523792892902393}, evalInputDelta = -0.006508709600943519
F(3.355120340022222) = LineSearchPoint{point=PointSample{avg=8.555493863739075}, derivative=-0.00457214976232486}, evalInputDelta = 0.0018755874284703822
F(0.25808618000170935) = LineSearchPoint{point=PointSample{avg=8.550759428023497}, derivative=-0.00968470184600782}, evalInputDelta = -0.002858848287107918
F(1.8066032600119655) = LineSearchPoint{point=PointSample{avg=8.559991333301623}, derivative=-0.00208043736493341}, evalInputDelta = 0.006373056991018089
F(0.13896948153938196) = LineSearchPoint{point=PointSample{avg=8.551618637061022}, derivative=-0.004661636713433075}, evalInputDelta = -0.0019996392495826854
F(0.9727863707756738) = LineSearchPoint{point=PointSample{avg=8.56149812392583}, derivative=-0.0014935109427359626}, evalInputDelta = 0.0078798476152250
...skipping 46256 bytes...
et at 6.181048925079882E-5
Converged to right
Fitness changed from 8.53089067251659 to 8.5308896110264
Iteration 34 complete. Error: 8.5308896110264 Total: 0.2207; Orientation: 0.0028; Line Search: 0.2077
F(0.0) = LineSearchPoint{point=PointSample{avg=8.5308896110264}, derivative=-0.01847473146732505}
New Minimum: 8.5308896110264 > 8.53088876783814
F(6.181048925079882E-5) = LineSearchPoint{point=PointSample{avg=8.53088876783814}, derivative=-0.00875954997093206}, evalInputDelta = -8.431882587700557E-7
F(4.326734247555917E-4) = LineSearchPoint{point=PointSample{avg=8.530897186659953}, derivative=0.0561294124655284}, evalInputDelta = 7.57563355335833E-6
F(3.328257113504552E-5) = LineSearchPoint{point=PointSample{avg=8.530889082356296}, derivative=-0.013279829073924668}, evalInputDelta = -5.286701032503061E-7
F(2.3297799794531863E-4) = LineSearchPoint{point=PointSample{avg=8.530889672333107}, derivative=0.0197254658899185}, evalInputDelta = 6.13067072663398E-8
F(1.79213844573322E-5) = LineSearchPoint{point=PointSample{avg=8.53088930486896}, derivative=-0.015687935039989803}, evalInputDelta = -3.061574389562338E-7
New Minimum: 8.53088876783814 > 8.530888536851318
F(1.2544969120132542E-4) = LineSearchPoint{point=PointSample{avg=8.530888536851318}, derivative=0.0015536746032920167}, evalInputDelta = -1.0741750813281215E-6
8.530888536851318 <= 8.5308896110264
New Minimum: 8.530888536851318 > 8.530888529512843
F(1.1571811303563865E-4) = LineSearchPoint{point=PointSample{avg=8.530888529512843}, derivative=-4.42293422622201E-5}, evalInputDelta = -1.0815135560449107E-6
Left bracket at 1.1571811303563865E-4
Converged to left
Fitness changed from 8.5308896110264 to 8.530888529512843
Iteration 35 complete. Error: 8.530888529512843 Total: 0.7520; Orientation: 0.0030; Line Search: 0.7383
F(0.0) = LineSearchPoint{point=PointSample{avg=8.530888529512843}, derivative=-0.034513109511733454}
New Minimum: 8.530888529512843 > 8.530888236724264
F(1.1571811303563865E-4) = LineSearchPoint{point=PointSample{avg=8.530888236724264}, derivative=0.028350722558132598}, evalInputDelta = -2.9278857915926437E-7
8.530888236724264 <= 8.530888529512843
New Minimum: 8.530888236724264 > 8.530887469963043
F(6.35308376245899E-5) = LineSearchPoint{point=PointSample{avg=8.530887469963043}, derivative=8.170101662914476E-4}, evalInputDelta = -1.0595498007859305E-6
Right bracket at 6.35308376245899E-5
New Minimum: 8.530887469963043 > 8.530887469345966
F(6.206168493885512E-5) = LineSearchPoint{point=PointSample{avg=8.530887469345966}, derivative=2.2859524867403086E-5}, evalInputDelta = -1.060166876953872E-6
Right bracket at 6.206168493885512E-5
Converged to right
Fitness changed from 8.530888529512843 to 8.530887469345966
Iteration 36 complete. Error: 8.530887469345966 Total: 0.7354; Orientation: 0.0028; Line Search: 0.7237
F(0.0) = LineSearchPoint{point=PointSample{avg=8.530887469345966}, derivative=-0.018584135894362766}
New Minimum: 8.530887469345966 > 8.530886621140993
F(6.206168493885512E-5) = LineSearchPoint{point=PointSample{avg=8.530886621140993}, derivative=-0.008700007662785423}, evalInputDelta = -8.482049729252594E-7
F(4.3443179457198583E-4) = LineSearchPoint{point=PointSample{avg=8.530895309963347}, derivative=0.05739446092303155}, evalInputDelta = 7.840617380239223E-6
F(3.3417830351691216E-5) = LineSearchPoint{point=PointSample{avg=8.530886936367565}, derivative=-0.013299289653631686}, evalInputDelta = -5.329784009688865E-7
F(2.3392481246183852E-4) = LineSearchPoint{point=PointSample{avg=8.530887582654614}, derivative=0.02029818401178024}, evalInputDelta = 1.1330864779779404E-7
F(1.7994216343218348E-5) = LineSearchPoint{point=PointSample{avg=8.530887160407746}, derivative=-0.01574920291555271}, evalInputDelta = -3.089382207832614E-7
New Minimum: 8.530886621140993 > 8.530886398810912
F(1.2595951440252844E-4) = LineSearchPoint{point=PointSample{avg=8.530886398810912}, derivative=0.0017959975670566047}, evalInputDelta = -1.070535054381594E-6
8.530886398810912 <= 8.530887469345966
New Minimum: 8.530886398810912 > 8.530886389135771
F(1.1485934266700845E-4) = LineSearchPoint{point=PointSample{avg=8.530886389135771}, derivative=-5.107664518390863E-5}, evalInputDelta = -1.080210195070208E-6
Left bracket at 1.1485934266700845E-4
Converged to left
Fitness changed from 8.530887469345966 to 8.530886389135771
Iteration 37 complete. Error: 8.530886389135771 Total: 0.4842; Orientation: 0.0028; Line Search: 0.4718
F(0.0) = LineSearchPoint{point=PointSample{avg=8.530886389135771}, derivative=-0.03430843613303743}
New Minimum: 8.530886389135771 > 8.530886058471733
F(1.1485934266700845E-4) = LineSearchPoint{point=PointSample{avg=8.530886058471733}, derivative=0.02748102977922213}, evalInputDelta = -3.3066403837267444E-7
8.530886058471733 <= 8.530886389135771
New Minimum: 8.530886058471733 > 8.530885331113993
F(6.377534364464969E-5) = LineSearchPoint{point=PointSample{avg=8.530885331113993}, derivative=7.907988316649007E-4}, evalInputDelta = -1.0580217786326784E-6
Right bracket at 6.377534364464969E-5
New Minimum: 8.530885331113993 > 8.53088533052985
F(6.233846140793043E-5) = LineSearchPoint{point=PointSample{avg=8.53088533052985}, derivative=2.2111024020925825E-5}, evalInputDelta = -1.0586059211448173E-6
Right bracket at 6.233846140793043E-5
Converged to right
Fitness changed from 8.530886389135771 to 8.53088533052985
Iteration 38 complete. Error: 8.53088533052985 Total: 0.6456; Orientation: 0.1543; Line Search: 0.4826
F(0.0) = LineSearchPoint{point=PointSample{avg=8.53088533052985}, derivative=-0.01870120916005206}
New Minimum: 8.53088533052985 > 8.530884476989062
F(6.233846140793043E-5) = LineSearchPoint{point=PointSample{avg=8.530884476989062}, derivative=-0.008631228323773462}, evalInputDelta = -8.535407882703794E-7
F(4.36369229855513E-4) = LineSearchPoint{point=PointSample{avg=8.530893466686601}, derivative=0.058792216016816626}, evalInputDelta = 8.136156750992996E-6
F(3.3566863835039466E-5) = LineSearchPoint{point=PointSample{avg=8.530884792897734}, derivative=-0.013317432566786613}, evalInputDelta = -5.376321166039588E-7
F(2.3496804684527625E-4) = LineSearchPoint{point=PointSample{avg=8.530885502339812}, derivative=0.020932130521403525}, evalInputDelta = 1.7180996181309638E-7
F(1.8074465141944327E-5) = LineSearchPoint{point=PointSample{avg=8.530885018575097}, derivative=-0.015813334166466797}, evalInputDelta = -3.119547535845868E-7
New Minimum: 8.530884476989062 > 8.530884264483799
F(1.2652125599361028E-4) = LineSearchPoint{point=PointSample{avg=8.530884264483799}, derivative=0.0020659693456743606}, evalInputDelta = -1.066046051434455E-6
8.530884264483799 <= 8.53088533052985
New Minimum: 8.530884264483799 > 8.530884251865213
F(1.1393461422198294E-4) = LineSearchPoint{point=PointSample{avg=8.530884251865213}, derivative=-5.8686767583325094E-5}, evalInputDelta = -1.0786646367222374E-6
Left bracket at 1.1393461422198294E-4
Converged to left
Fitness changed from 8.53088533052985 to 8.530884251865213
Iteration 39 complete. Error: 8.530884251865213 Total: 0.6746; Orientation: 0.0031; Line Search: 0.6484
F(0.0) = LineSearchPoint{point=PointSample{avg=8.530884251865213}, derivative=-0.03408150852772551}
New Minimum: 8.530884251865213 > 8.530883881654558
F(1.1393461422198294E-4) = LineSearchPoint{point=PointSample{avg=8.530883881654558}, derivative=0.02654756903305593}, evalInputDelta = -3.702106550207418E-7
8.530883881654558 <= 8.530884251865213
New Minimum: 8.530883881654558 > 8.53088319562226
F(6.404622472305976E-5) = LineSearchPoint{point=PointSample{avg=8.53088319562226}, derivative=7.626961224203941E-4}, evalInputDelta = -1.0562429526572714E-6
Right bracket at 6.404622472305976E-5
New Minimum: 8.53088319562226 > 8.530883195072613
F(6.264433285201835E-5) = LineSearchPoint{point=PointSample{avg=8.530883195072613}, derivative=2.1308243230577437E-5}, evalInputDelta = -1.056792600095946E-6
Right bracket at 6.264433285201835E-5
Converged to right
Fitness changed from 8.530884251865213 to 8.530883195072613
Iteration 40 complete. Error: 8.530883195072613 Total: 0.5234; Orientation: 0.0026; Line Search: 0.5005
Final threshold in iteration 40: 8.530883195072613 (> 0.0) after 30.281s (< 30.000s)
Returns
8.530883195072613
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.3603051934727292 ], [ 0.7766267928313344 ], [ 0.5004949277077191 ], [ -0.567604295484103 ], [ -0.24575576717907066 ], [ -0.7352731424615937 ], [ -0.07668281805021615 ], [ 0.8353361792623836 ], ... ],
[ [ 0.5414332382875104 ], [ -0.11958585309344971 ], [ 0.25422888513482184 ], [ 0.22071537526894652 ], [ -0.8739393093920977 ], [ -0.1979201331861813 ], [ -0.7174800334829694 ], [ 0.2280994047111287 ], ... ],
[ [ -0.6679796127011541 ], [ 0.2699484799952876 ], [ 0.6826296182877393 ], [ 0.3909806560878376 ], [ 0.9241815069947509 ], [ 0.7099799061983123 ], [ -0.9075258636514625 ], [ -0.46057625089960097 ], ... ],
[ [ -0.23215044345743707 ], [ -0.3890998653356017 ], [ -0.9785746064056067 ], [ 0.22379164860602566 ], [ 0.5580005503177193 ], [ 0.7591211819275366 ], [ 0.290921593218895 ], [ 0.9789976178604112 ], ... ],
[ [ 0.3187085202362482 ], [ 0.18281225237309237 ], [ -0.4611774915116107 ], [ 0.29643986744460055 ], [ -0.26645366817651706 ], [ -0.9233934703725706 ], [ -0.6031859486728848 ], [ 0.7469668097139015 ], ... ],
[ [ 0.3645154490825091 ], [ -0.4153968808034918 ], [ -0.9742419893501422 ], [ 0.2461235517657267 ], [ -0.8716834275485836 ], [ -0.9817172049762419 ], [ 0.9853904555397677 ], [ 0.28406640628128366 ], ... ],
[ [ 0.2775693084548022 ], [ -0.7664397461318597 ], [ -0.6227997209767129 ], [ -0.5122768236246127 ], [ 0.18880567435437695 ], [ 0.31238107928150743 ], [ 0.6650728085171611 ], [ 0.3025841289366888 ], ... ],
[ [ -0.49057723858683744 ], [ 0.9722219645613229 ], [ 0.35229778859009353 ], [ -0.48238711527764155 ], [ 0.11723107368719457 ], [ 0.6461047463523318 ], [ 0.21449478795654717 ], [ 0.41644530885652814 ], ... ],
...
]
To produce the following output:
TrainingTester.java:633 executed in 0.01 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ 1.6659606491627705 ], [ 1.1347333405180795 ], [ 1.4135141480928943 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 1.0941310784944611 ], ... ],
[ [ 1.3590253049050989 ], [ 0.0 ], [ 1.9832960801123303 ], [ 2.1285492643322885 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 2.0938131550312304 ], ... ],
[ [ 0.0 ], [ 1.9246845352377613 ], [ 1.2103401387673263 ], [ 1.599272113338872 ], [ 1.0402108107384467 ], [ 1.1867984521808042 ], [ 0.0 ], [ 0.0 ], ... ],
[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 2.1138689627310185 ], [ 1.3386982213704994 ], [ 1.147742451303396 ], [ 1.8540097826001531 ], [ 1.0106695528660181 ], ... ],
[ [ 1.771345029690202 ], [ 2.3388229688343216 ], [ 0.0 ], [ 1.8366723787669705 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 1.157042595654558 ], ... ],
[ [ 1.6563115495572407 ], [ 0.0 ], [ 0.0 ], [ 2.015688477240838 ], [ 0.0 ], [ 0.0 ], [ 1.0073857988499682 ], [ 1.8762472187321357 ], ... ],
[ [ 1.8980789740137616 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 2.30140196086824 ], [ 1.7891948501332475 ], [ 1.2262115540064689 ], [ 1.8179290416444995 ], ... ],
[ [ 0.0 ], [ 1.0141852400652533 ], [ 1.6847871500155784 ], [ 0.0 ], [ 2.9206440340469157 ], [ 1.2440806471865427 ], [ 2.159193818259022 ], [ 1.5496050137736694 ], ... ],
...
]
Next, we apply the same optimization using L-BFGS, which is nearly ideal for purely second-order or quadratic functions.
TrainingTester.java:509 executed in 33.40 seconds (0.067 gc):
IterativeTrainer iterativeTrainer = new IterativeTrainer(trainable.addRef());
try {
iterativeTrainer.setLineSearchFactory(label -> new ArmijoWolfeSearch());
iterativeTrainer.setOrientation(new LBFGS());
iterativeTrainer.setMonitor(TrainingTester.getMonitor(history));
iterativeTrainer.setTimeout(30, TimeUnit.SECONDS);
iterativeTrainer.setIterationsPerSample(100);
iterativeTrainer.setMaxIterations(250);
iterativeTrainer.setTerminateThreshold(0);
return iterativeTrainer.run();
} finally {
iterativeTrainer.freeRef();
}
Reset training subject: 2830359283123
Reset training subject: 2830371045720
Adding measurement 55cb3b91 to history. Total: 0
LBFGS Accumulation History: 1 points
Constructing line search parameters: GD
Non-optimal measurement 15.162946939207576 < 15.162946939207576. Total: 1
th(0)=15.162946939207576;dx=-5.645340560031366E9
Adding measurement 1a7a4af1 to history. Total: 1
New Minimum: 15.162946939207576 > 8.870446379707042
Armijo: th(2.154434690031884)=8.870446379707042; dx=-0.06121239539696214 evalInputDelta=6.292500559500533
Non-optimal measurement 8.950114943395853 < 8.870446379707042. Total: 2
Armijo: th(1.077217345015942)=8.950114943395853; dx=-0.12530897402939326 evalInputDelta=6.212831995811722
Non-optimal measurement 9.064252424443843 < 8.870446379707042. Total: 2
Armijo: th(0.3590724483386473)=9.064252424443843; dx=-0.40437638976051327 evalInputDelta=6.098694514763732
Non-optimal measurement 9.262985820599429 < 8.870446379707042. Total: 2
Armijo: th(0.08976811208466183)=9.262985820599429; dx=-1.6120565185253828 evalInputDelta=5.899961118608147
Non-optimal measurement 9.458966290987814 < 8.870446379707042. Total: 2
Armijo: th(0.017953622416932366)=9.458966290987814; dx=-10.1448197287502 evalInputDelta=5.703980648219762
Non-optimal measurement 9.745679166317343 < 8.870446379707042. Total: 2
Armijo: th(0.002992270402822061)=9.745679166317343; dx=-50.77172788049012 evalInputDelta=5.4172677728902325
Non-optimal measurement 10.033227448182458 < 8.870446379707042. Total: 2
END: th(4.2746720040315154E-4)=10.033227448182458; dx=-338.3238798728223 evalInputDelta=5.129719491025117
Fitness changed from 15.162946939207576 to 8.870446379707042
Iteration 1 complete. Error: 8.870446379707042 Total: 0.8262; Orientation: 0.1156; Line Search: 0.5760
Non-optimal measurement 8.870446379707042 < 8.870446379707042. Total: 2
LBFGS Accumulation History: 2 points
Non-optimal measurement 8.870446379707042 < 8.870446379707042. Total: 2
th(0)=8.870446379707042;dx=-0.026583894433485716
Adding measurement 743d49a6 to history. Total: 2
New Minimum: 8.870446379707042 > 8.870421894620833
WOLFE (weak): th(9.20950165399361E-4)=8.870421894620833; dx=-0.026589640220459537 evalInputDelta=2.448508620922496E-5
Adding measurement 512dd618 to history. Total: 3
New Minimum: 8.870421894620833 > 8.870397404233682
WOLFE (weak): th(0.001841900330798722)=8.870397404233682; dx=-0.026595406400326545 evalInputDelta=4.897547336035757E-5
Adding measurement 2b608386 to history. Total: 4
New Minimum: 8.870397404233682 > 8.870299389297763
WOLFE (weak): th(0.005525700992396166)=8.870299389297763; dx=-0.02661867672048679 evalInputDelta=1.4699040927901308E-4
Adding measurement 583a1f86 to history. Total: 5
New Minimum: 8.870299389297763 > 8.869857235645897
WOLFE (weak): th(0.022102803969584663)=8.869857235645897; dx=-0.026727595252618357 evalInputDelta=5.891440611449639E-4
Adding measurement 13e383f9 to history. Total: 6
New Minimum: 8.869857235645897 > 8.867464310859116
WOLFE (weak): th(0.11051401984792332)=8.867464310859116; dx=-0.027446845262080466 evalInputDelta=0.002982068847925845
Adding measurement 7f3819a1 to history. Total: 7
New Minimum: 8.867464310859116 > 8.84518362635804
WOLFE (weak): th(0.66308411908754)=8.84518362635804; dx=-0.10172704419204949 evalInputDelta=0.02526275334900241
Adding measurement 737ddc82 to history. Total: 8
New Minimum: 8.84518362635804 > 8.798367489241965
END: th(4.641588833612779)=8.798367489241965; dx=-0.015125659667823465 evalInputDelta=0.0720788904650771
Fitness changed from 8.870446379707042 to 8.798367489241965
Iteration 2 complete. Error: 8.798367489241965 Total: 0.5790; Orientation: 0.0039; Line Search: 0.5652
Non-optimal measurement 8.798367489241965 < 8.798367489241965. Total: 9
Rejected: LBFGS Orientation magnitude: 1.787e+00, gradient 1.926e-01, dot -0.684; [57e53556-91a2-4ee3-bcaa-f10c078466e9 = 1.000/1.000e+00, 48c6dc1f-10b0-4d2d-90fd-5d5c655c75dc = 1.000/1.000e+00, a4e19ac1-ff9c-4533-9c99-2ba89476ffdd = 1.000/1.000e+00, eace70da-f5e4-4948-9ad8-9105f85cd0c0 = 1.000/1.000e+00, 7acadf33-dca2-40ba-9f61-cbf917f634bf = 1.000/1.000e+00]
Orientation rejected. Popping history element from 8.798367489241965, 8.84518362635804, 8.867464310859116, 8.869857235645897, 8.870299389297763, 8.870397404233682, 8.870421894620833, 8.870446379707042, 15.162946939207576
Accepted: LBFGS Orientation magnitude: 8.176e-01, gradient 1.926e-01, dot 0.583; [48c6dc1f-10b0-4d2d-90fd-5d5c655c75dc = 1.000/1.000e+00, 57e53556-91a2-4ee3-bcaa-f10c078466e9 = 1.000/1.000e+00, 7acadf33-dca2-40ba-9f61-cbf917f634bf = 1.000/1.000e+00, a4e19ac1-ff9c-4533-9c99-2ba89476ffdd = 1.000/1.000e+00, eace70da-f5e4-4948-9ad8-9105f85cd0c0 = 1.000/1.000e+00]
Overwriting history with 8 points
Constructing line search parameters: LBFGS
Non-optimal measurement 8.798367489241965 < 8.798367489241965. Total: 8
th(0)=8.798367489241965;dx=-0.09180714939958354
Adding measurement 6fcf2cce to history. Total: 8
New Minimum: 8.798367489241965 > 8.70371287253339
END: th(2.154434690031884)=8.70371287253339; dx=-0.04571190485561426 evalInputDelta=0.09465461670857422
Fitness changed from 8.798367489241965 to 8.70371287253339
Iteration 3 complete. Error: 8.70371287253339 Total: 4.3889; Orientation: 4.3235; Line Search: 0.0556
Non-optimal measurement 8.70371287253339 < 8.70371287253339. Total: 9
Accepted: LBFGS Orientation magnitude: 1.778e-01, gradient 1.302e-01, dot 0.660; [57e53556-91a2-4ee3-bcaa-f10c078466e9 = 1.000/1.000e+00, 48c6dc1f-10b0-4d2d-90fd-5d5c655c75dc = 1.000/1.000e+00, 7acadf33-dca2-40ba-9f61-cbf917f634bf = 1.000/1.000e+00, a4e19ac1-ff9c-4533-9c99-2ba89476ffdd = 1.000/1.000e+00, eace70da-f5e4-4948-9ad8-9105f85cd0c0 = 1.000/1.000e+00]
Non-optimal measurement 8.70371287253339 < 8.70371287253339. Total: 9
th(0)=8.70371287253339;dx=-0.015283870800750462
Adding measurement 5c2bf024 to history. Total: 9
New Minimum: 8.70371287253339 > 8.68113872564559
WOLFE (weak): th(4.641588833612779)=8.68113872564559; dx=-0.014587640420036543 evalInputDelta=0.02257414688780024
Adding measurement eaa420c to history. Total: 10
New Minimum: 8.68113872564559 > 8.678818746838598
END: th(9.283177667225559)=8.678818746838598; dx=-0.005382759966248138 evalInputDelta=0.02489412569479299
Fitness changed from 8.70371287253339 to 8.678818746838598
Iteration 4 complete. Error: 8.678818746838598 Total: 2.4993; Orientation: 2.3412; Line Search: 0.0658
Non-optimal measurement 8.678818746838598 < 8.678818746838598. Total: 11
Accepted: LBFGS Orientation magnitude: 1.075e+02, gradient 8.331e-01, dot 0.047; [48c6dc1f-10b0-4d2d-90fd-5d5c655c75dc = 1.000/1.000e+00, a4e19ac1-ff9c-4533-9c99-2ba89476ffdd = 1.000/1.000e+00, 57e53556-91a2-4ee3-bcaa-f10c078466e9 = 1.000/1.000e+00, eace70da-f5e4-4948-9ad8-9105f85cd0c0 = 1.000/1.000e+00, 7acadf33-dca2-40ba-9f61-cbf917f634bf = 1.000/1.000e+00]
Non-optimal measurement 8.678818746838598 < 8.678818746838598. Total: 11
th(0)=8.678818746838598;dx=-4.242468027806788
Non-optimal measurement 8.702451735378082 < 8.678818746838598. Total: 11
Armijo: th(20.000000000000004)=8.702451735378082; dx=1.6459745935613288 evalInputDelta=-0.023632988539484145
Adding measurement 817ebcd to history. Total: 11
New Minimum: 8.678818746838598 > 8.639020853093319
WOLF (strong): th(10.000000000000002)=8.639020853093319; dx=6.054433841811833 evalInputDelta=0.039797893745278756
Adding measurement 226cccf6 to history. Total: 12
New Minimum: 8.639020853093319 > 8.561479230517065
WOLF (strong): th(3.333333333333334)=8.561479230517065; dx=21.153381892349053 evalInputDelta=0.1173395163215325
Non-optimal measurement 8.961580130635152 < 8.561479230517065. Total: 13
Armijo: th(0.8333333333333335)=8.961580130635152; dx=5318.96806526385 evalInputDelta=-0.28276138379655436
Adding measurement 2b5b78dc to history. Total: 13
New Minimum: 8.561479230517065 > 8.526143474220657
WOLF (strong): th(0.16666666666666669)=8.526143474220657; dx=3.911870873846787 evalInputDelta=0.152675272617941
Non-optimal measurement 8.678723794417968 < 8.526143474220657. Total: 14
WOLF (strong): th(0.02777777777777778)=8.678723794417968; dx=404.2919570464566 evalInputDelta=9.495242063017884E-5
Non-optimal measurement 8.663146588320037 < 8.526143474220657. Total: 14
END: th(0.003968253968253969)=8.663146588320037; dx=-3.7921355864705437 evalInputDelta=0.01567215851856041
Fitness changed from 8.678818746838598 to 8.526143474220657
Iteration 5 complete. Error: 8.526143474220657 Total: 2.6786; Orientation: 1.9142; Line Search: 0.5702
Non-optimal measurement 8.526143474220657 < 8.526143474220657. Total: 14
Accepted: LBFGS Orientation magnitude: 1.819e+05, gradient 9.853e+00, dot 0.004; [a4e19ac1-ff9c-4533-9c99-2ba89476ffdd = 1.000/1.000e+00, eace70da-f5e4-4948-9ad8-9105f85cd0c0 = 1.000/1.000e+00, 57e53556-91a2-4ee3-bcaa-f10c078466e9 = 1.000/1.000e+00, 7acadf33-dca2-40ba-9f61-cbf917f634bf = 1.000/1.000e+00, 48c6dc1f-10b0-4d2d-90fd-5d5c655c75dc = 1.000/1.000e+00]
Non-optimal measurement 8.526143474220657 < 8.526143474220657. Total: 14
th(0)=8.526143474220657;dx=-7988.561623092591
Non-optimal measurement 9.42787679532932 < 8.526143474220657. Total: 14
Armijo: th(0.008549344008063033)=9.42787679532932; dx=74329.8630806015 evalInputDelta=-0.9017333211086633
Non-optimal measurement 9.495594115447108 < 8.526143474220657. Total: 14
Armijo: th(0.004274672004031517)=9.495594115447108; dx=108078.67610249593 evalInputDelta=-0.9694506412264516
Non-optimal measurement 9.493663271806492 < 8.526143474220657. Total: 14
Armijo: th(0.0014248906680105055)=9.493663271806492; dx=343466.068101763 evalInputDelta=-0.967519797585835
Non-optimal measurement 10.211501871234162 < 8.526143474220657. Total: 14
Armijo: th(3.5622266700262637E-4)=10.211501871234162; dx=4544753.14327629 evalInputDelta=-1.6853583970135055
Non-optimal measurement 8.67889033884118 < 8.526143474220657. Total: 14
Armijo: th(7.124453340052527E-5)=8.67889033884118; dx=-69617.09312071011 evalInputDelta=-0.15274686462052323
Adding measurement 10d4953f to history. Total: 14
New Minimum: 8.526143474220657 > 8.510870196873027
WOLFE (weak): th(1.1874088900087545E-5)=8.510870196873027; dx=-13135.553498816216 evalInputDelta=0.015273277347629843
Non-optimal measurement 8.54864101104074 < 8.510870196873027. Total: 15
Armijo: th(4.155931115030641E-5)=8.54864101104074; dx=4209.917710104238 evalInputDelta=-0.02249753682008304
Non-optimal measurement 8.549215442979918 < 8.510870196873027. Total: 15
Armijo: th(2.6716700025196975E-5)=8.549215442979918; dx=-15116.317293365515 evalInputDelta=-0.023071968759261097
Non-optimal measurement 8.518691367467289 < 8.510870196873027. Total: 15
WOLF (strong): th(1.929539446264226E-5)=8.518691367467289; dx=1820.16430247161 evalInputDelta=0.0074521067533677154
Non-optimal measurement 8.533983623574509 < 8.510870196873027. Total: 15
Armijo: th(1.5584741681364903E-5)=8.533983623574509; dx=-43797.69555982052 evalInputDelta=-0.0078401493538518
Adding measurement 48d691ad to history. Total: 15
New Minimum: 8.510870196873027 > 8.504311191090151
WOLF (strong): th(1.3729415290726223E-5)=8.504311191090151; dx=508.46188803367835 evalInputDelta=0.02183228313050556
Non-optimal measurement 8.5049428768781 < 8.504311191090151. Total: 16
END: th(1.2801752095406883E-5)=8.5049428768781; dx=-2369.6283922951598 evalInputDelta=0.021200597342556193
Fitness changed from 8.526143474220657 to 8.504311191090151
Iteration 6 complete. Error: 8.504311191090151 Total: 4.5438; Orientation: 3.2600; Line Search: 1.2630
Non-optimal measurement 8.504311191090151 < 8.504311191090151. Total: 16
Accepted: LBFGS Orientation magnitude: 1.222e+04, gradient 1.344e+00, dot 0.002; [48c6dc1f-10b0-4d2d-90fd-5d5c655c75dc = 1.000/1.000e+00, a4e19ac1-ff9c-4533-9c99-2ba89476ffdd = 1.000/1.000e+00, eace70da-f5e4-4948-9ad8-9105f85cd0c0 = 1.000/1.000e+00, 57e53556-91a2-4ee3-bcaa-f10c078466e9 = 1.000/1.000e+00, 7acadf33-dca2-40ba-9f61-cbf917f634bf = 1.000/1.000e+00]
Non-optimal measurement 8.504311191090151 < 8.504311191090151. Total: 16
th(0)=8.504311191090151;dx=-35.035753871054425
Non-optimal measurement 8.51059611168363 < 8.504311191090151. Total: 16
Armijo: th(2.7580538807532947E-5)=8.51059611168363; dx=843.5248001339555 evalInputDelta=-0.006284920593479626
Non-optimal measurement 8.504904689183041 < 8.504311191090151. Total: 16
Armijo: th(1.3790269403766473E-5)=8.504904689183041; dx=153.4485453511475 evalInputDelta=-5.934980928898881E-4
Adding measurement 2aaadf40 to history. Total: 16
New Minimum: 8.504311191090151 > 8.504243615387065
WOLF (strong): th(4.5967564679221575E-6)=8.504243615387065; dx=7.885714231076918 evalInputDelta=6.757570308657534E-5
Non-optimal measurement 8.50427632072461 < 8.504243615387065. Total: 17
END: th(1.1491891169805394E-6)=8.50427632072461; dx=-25.530160330076587 evalInputDelta=3.487036554084E-5
Fitness changed from 8.504311191090151 to 8.504243615387065
Iteration 7 complete. Error: 8.504243615387065 Total: 3.6551; Orientation: 3.2897; Line Search: 0.3561
Non-optimal measurement 8.504243615387065 < 8.504243615387065. Total: 17
Accepted: LBFGS Orientation magnitude: 3.065e+03, gradient 1.646e+00, dot 0.001; [7acadf33-dca2-40ba-9f61-cbf917f634bf = 1.000/1.000e+00, eace70da-f5e4-4948-9ad8-9105f85cd0c0 = 1.000/1.000e+00, 57e53556-91a2-4ee3-bcaa-f10c078466e9 = 1.000/1.000e+00, a4e19ac1-ff9c-4533-9c99-2ba89476ffdd = 1.000/1.000e+00, 48c6dc1f-10b0-4d2d-90fd-5d5c655c75dc = 1.000/1.000e+00]
Non-optimal measurement 8.504243615387065 < 8.504243615387065. Total: 17
th(0)=8.504243615387065;dx=-3.151254308179725
Adding measurement 4713680e to history. Total: 17
New Minimum: 8.504243615387065 > 8.504237983267675
END: th(2.4758528990299826E-6)=8.504237983267675; dx=-1.4114415504722668 evalInputDelta=5.632119389531454E-6
Fitness changed from 8.504243615387065 to 8.504237983267675
Iteration 8 complete. Error: 8.504237983267675 Total: 3.9909; Orientation: 3.7311; Line Search: 0.1968
Non-optimal measurement 8.504237983267675 < 8.504237983267675. Total: 18
Accepted: LBFGS Orientation magnitude: 6.580e+02, gradient 1.598e+00, dot 0.001; [48c6dc1f-10b0-4d2d-90fd-5d5c655c75dc = 1.000/1.000e+00, a4e19ac1-ff9c-4533-9c99-2ba89476ffdd = 1.000/1.000e+00, eace70da-f5e4-4948-9ad8-9105f85cd0c0 = 1.000/1.000e+00, 7acadf33-dca2-40ba-9f61-cbf917f634bf = 1.000/1.000e+00, 57e53556-91a2-4ee3-bcaa-f10c078466e9 = 1.000/1.000e+00]
Non-optimal measurement 8.504237983267675 < 8.504237983267675. Total: 18
th(0)=8.504237983267675;dx=-0.9726929149619801
Adding measurement 3ccdb3b1 to history. Total: 18
New Minimum: 8.504237983267675 > 8.504233298882955
END: th(5.334063373086202E-6)=8.504233298882955; dx=-0.7843943047004149 evalInputDelta=4.68438472012167E-6
Fitness changed from 8.504237983267675 to 8.504233298882955
Iteration 9 complete. Error: 8.504233298882955 Total: 4.5388; Orientation: 4.3732; Line Search: 0.1558
Non-optimal measurement 8.504233298882955 < 8.504233298882955. Total: 19
Accepted: LBFGS Orientation magnitude: 4.758e+02, gradient 1.576e+00, dot 0.001; [eace70da-f5e4-4948-9ad8-9105f85cd0c0 = 1.000/1.000e+00, 7acadf33-dca2-40ba-9f61-cbf917f634bf = 1.000/1.000e+00, a4e19ac1-ff9c-4533-9c99-2ba89476ffdd = 1.000/1.000e+00, 48c6dc1f-10b0-4d2d-90fd-5d5c655c75dc = 1.000/1.000e+00, 57e53556-91a2-4ee3-bcaa-f10c078466e9 = 1.000/1.000e+00]
Non-optimal measurement 8.504233298882955 < 8.504233298882955. Total: 19
th(0)=8.504233298882955;dx=-0.7891729148973576
Adding measurement 43ba18a9 to history. Total: 19
New Minimum: 8.504233298882955 > 8.504225071175364
END: th(1.1491891169805396E-5)=8.504225071175364; dx=-0.6420310709286381 evalInputDelta=8.227707590791056E-6
Fitness changed from 8.504233298882955 to 8.504225071175364
Iteration 10 complete. Error: 8.504225071175364 Total: 5.6969; Orientation: 5.5835; Line Search: 0.1039
Final threshold in iteration 10: 8.504225071175364 (> 0.0) after 33.398s (< 30.000s)
Returns
8.504225071175364
This training apply resulted in the following configuration:
TrainingTester.java:610 executed in 0.00 seconds (0.000 gc):
RefList<double[]> state = network.state();
assert state != null;
String description = state.stream().map(RefArrays::toString).reduce((a, b) -> a + "\n" + b)
.orElse("");
state.freeRef();
return description;
Returns
TrainingTester.java:622 executed in 0.00 seconds (0.000 gc):
return RefArrays.stream(RefUtil.addRef(data)).flatMap(x -> {
return RefArrays.stream(x);
}).limit(1).map(x -> {
String temp_18_0015 = x.prettyPrint();
x.freeRef();
return temp_18_0015;
}).reduce((a, b) -> a + "\n" + b).orElse("");
Returns
[
[ [ 0.368168693257317 ], [ 0.7782413148932116 ], [ 0.5010844336068497 ], [ -0.567604295484103 ], [ -0.24575576717907066 ], [ -0.7352731424615937 ], [ -0.07668281805021615 ], [ 0.8367303892364529 ], ... ],
[ [ 0.5421975318297766 ], [ -0.11958585309344971 ], [ 0.2721171952863878 ], [ 0.32143901173784684 ], [ -0.8739393093920977 ], [ -0.1979201331861813 ], [ -0.7174800334829694 ], [ 0.24851771424766775 ], ... ],
[ [ -0.6679796127011541 ], [ 0.28524369250764375 ], [ 0.6847242553386564 ], [ 0.3975791476890848 ], [ 0.925319370166855 ], [ 0.704553213869658 ], [ -0.9075258636514625 ], [ -0.46057625089960097 ], ... ],
[ [ -0.23215044345743707 ], [ -0.3890998653356017 ], [ -0.9785746064056067 ], [ 0.2504673852683376 ], [ 0.5611545049781231 ], [ 0.760811612859768 ], [ 0.3036601692553411 ], [ 0.9762499286768168 ], ... ],
[ [ 0.3290221820940171 ], [ 0.2011921068734048 ], [ -0.4611774915116107 ], [ 0.3086254311975398 ], [ -0.26645366817651706 ], [ -0.9233934703725706 ], [ -0.6031859486728848 ], [ 0.7487131561568087 ], ... ],
[ [ 0.3660277257520779 ], [ -0.4153968808034918 ], [ -0.9742419893501422 ], [ 0.29314975723924264 ], [ -0.8716834275485836 ], [ -0.9817172049762419 ], [ 0.985206747069763 ], [ 0.29755393292283144 ], ... ],
[ [ 0.273953842524253 ], [ -0.7664397461318597 ], [ -0.6227997209767129 ], [ -0.5122768236246127 ], [ 0.20941980446352496 ], [ 0.31620216486374564 ], [ 0.6654707145727818 ], [ 0.3042178112647165 ], ... ],
[ [ -0.49057723858683744 ], [ 0.9732497357205293 ], [ 0.3605552135519103 ], [ -0.48238711527764155 ], [ 0.10363185744494186 ], [ 0.6460289380790002 ], [ 0.24725894499673914 ], [ 0.42221957203020255 ], ... ],
...
]
To produce the following output:
TrainingTester.java:633 executed in 0.01 seconds (0.000 gc):
Result[] array = ConstantResult.batchResultArray(pop(RefUtil.addRef(data)));
@Nullable
Result eval = layer.eval(array);
assert eval != null;
TensorList tensorList = Result.getData(eval);
String temp_18_0016 = tensorList.stream().limit(1).map(x -> {
String temp_18_0017 = x.prettyPrint();
x.freeRef();
return temp_18_0017;
}).reduce((a, b) -> a + "\n" + b).orElse("");
tensorList.freeRef();
return temp_18_0016;
Returns
[
[ [ 1.6480734837201545 ], [ 1.1335556831848177 ], [ 1.4126824318232687 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 1.0932191452956523 ], ... ],
[ [ 1.3580671111628801 ], [ 0.0 ], [ 1.9169995325696279 ], [ 1.7638055625581006 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 2.005955640014846 ], ... ],
[ [ 0.0 ], [ 1.8723712979685596 ], [ 1.2084874476124052 ], [ 1.585945287591297 ], [ 1.0395710415366932 ], [ 1.1913602340158755 ], [ 0.0 ], [ 0.0 ], ... ],
[ [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 1.9981330762371943 ], [ 1.3349308606253227 ], [ 1.1464666701384452 ], [ 1.8147052059700006 ], [ 1.0120908357093428 ], ... ],
[ [ 1.7433613392441907 ], [ 2.2294335413866775 ], [ 0.0 ], [ 1.8000482445675199 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 1.1556924278210956 ], ... ],
[ [ 1.6528864075538936 ], [ 0.0 ], [ 0.0 ], [ 1.8469503913997845 ], [ 0.0 ], [ 0.0 ], [ 1.0074797165368021 ], [ 1.8332308310037935 ], ... ],
[ [ 1.910562731261635 ], [ 0.0 ], [ 0.0 ], [ 0.0 ], [ 2.185199664297891 ], [ 1.7783513973416925 ], [ 1.2258449037924153 ], [ 1.8130412336724806 ], ... ],
[ [ 0.0 ], [ 1.0136495986812097 ], [ 1.6653829348067208 ], [ 0.0 ], [ 3.1063712554660756 ], [ 1.2441536383744003 ], [ 2.011055212369112 ], [ 1.5389723580235148 ], ... ],
...
]
TrainingTester.java:432 executed in 0.10 seconds (0.000 gc):
return TestUtil.compare(title + " vs Iteration", runs);
Plotting range=[1.0, 0.9296347456210736], [98.0, 0.9507363654958303]; valueStats=DoubleSummaryStatistics{count=148, sum=1268.687411, min=8.504225, average=8.572212, max=8.927634}
Plotting 98 points for GD
Plotting 40 points for CjGD
Plotting 10 points for LBFGS
Returns
TrainingTester.java:435 executed in 0.01 seconds (0.000 gc):
return TestUtil.compareTime(title + " vs Time", runs);
Plotting range=[0.0, 0.9296347456210736], [32.571, 0.9507363654958303]; valueStats=DoubleSummaryStatistics{count=148, sum=1268.687411, min=8.504225, average=8.572212, max=8.927634}
Plotting 98 points for GD
Plotting 40 points for CjGD
Plotting 10 points for LBFGS
Returns
TrainingTester.java:255 executed in 0.00 seconds (0.000 gc):
return grid(inputLearning, modelLearning, completeLearning);
Returns
TrainingTester.java:258 executed in 0.00 seconds (0.000 gc):
return new ComponentResult(null == inputLearning ? null : inputLearning.value,
null == modelLearning ? null : modelLearning.value, null == completeLearning ? null : completeLearning.value);
Returns
{"input":{ "LBFGS": { "type": "NonConverged", "value": 8.504225071175364 }, "CjGD": { "type": "NonConverged", "value": 8.530883195072613 }, "GD": { "type": "NonConverged", "value": 8.570014981298746 } }, "model":null, "complete":null}
LayerTests.java:425 executed in 0.00 seconds (0.000 gc):
throwException(exceptions.addRef());
| details | result |
|---|---|
| {"input":{ "LBFGS": { "type": "NonConverged", "value": 8.504225071175364 }, "CjGD": { "type": "NonConverged", "value": 8.530883195072613 }, "GD": { "type": "NonConverged", "value": 8.570014981298746 } }, "model":null, "complete":null} | OK |
{
"result": "OK",
"performance": {
"execution_time": "94.651",
"gc_time": "0.573"
},
"created_on": 1586737357605,
"file_name": "trainingTest",
"report": {
"simpleName": "InvSqrtPowerTest",
"canonicalName": "com.simiacryptus.mindseye.layers.java.NthPowerActivationLayerTest.InvSqrtPowerTest",
"link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/test/java/com/simiacryptus/mindseye/layers/java/NthPowerActivationLayerTest.java",
"javaDoc": ""
},
"training_analysis": {
"input": {
"LBFGS": {
"type": "NonConverged",
"value": 8.504225071175364
},
"CjGD": {
"type": "NonConverged",
"value": 8.530883195072613
},
"GD": {
"type": "NonConverged",
"value": 8.570014981298746
}
}
},
"archive": "s3://code.simiacrypt.us/tests/com/simiacryptus/mindseye/layers/java/NthPowerActivationLayer/InvSqrtPowerTest/trainingTest/202004132237",
"id": "76250372-50ef-4163-8c90-0afaa4d6685d",
"report_type": "Components",
"display_name": "Comparative Training",
"target": {
"simpleName": "NthPowerActivationLayer",
"canonicalName": "com.simiacryptus.mindseye.layers.java.NthPowerActivationLayer",
"link": "https://github.com/SimiaCryptus/mindseye-java/tree/93db34cedee48c0202777a2b25deddf1dfaf5731/src/main/java/com/simiacryptus/mindseye/layers/java/NthPowerActivationLayer.java",
"javaDoc": ""
}
}