{ "cells": [ { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "import datetime\n", "\n", "from util.util import importstr\n", "from util.logconf import logging\n", "log = logging.getLogger('nb')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def run(app, *argv):\n", " argv.insert(0, '--num-workers=6') # <1>\n", " log.info(\"Running: {}({!r}).main()\".format(app, argv))\n", " \n", " app_cls = importstr(*app.rsplit('.', 1)) # <2>\n", " app_cls(argv).main()\n", " \n", " log.info(\"Finished: {}.{!r}).main()\".format(app, arg_list))" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "import os\n", "import shutil\n", "\n", "# clean up any old data that might be around.\n", "# We don't call this by default because it's destructive, \n", "# and would waste a lot of time if it ran when nothing \n", "# on the application side had changed.\n", "def cleanCache():\n", " shutil.rmtree('data-unversioned/cache')\n", " os.mkdir('data-unversioned/cache')\n", "\n", "cleanCache()" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "training_epochs = 20\n", "experiment_epochs = 10\n", "final_epochs = 50\n", "\n", "training_epochs = 2\n", "experiment_epochs = 2\n", "final_epochs = 5\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Chapter 11" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "2019-08-18 13:26:12,563 INFO pid:6276 nb:008:run Running: p2ch10.prepcache.LunaPrepCacheApp(None).main()\n", "2019-08-18 13:26:12,564 INFO pid:6276 p2ch10.prepcache:043:main Starting LunaPrepCacheApp, Namespace(batch_size=1024, num_workers=8)\n", "2019-08-18 13:26:16,158 INFO pid:6276 p2ch10.dsets:165:__init__ : 551065 training samples\n", "2019-08-18 13:26:16,159 WARNING pid:6276 util.util:248:enumerateWithEstimate Stuffing cache ----/539, starting\n", "2019-08-18 13:27:27,582 INFO pid:6276 util.util:272:enumerateWithEstimate Stuffing cache 16/539, done at 2019-08-18 13:52:18, 0:25:16\n", "2019-08-18 13:28:10,351 INFO pid:6276 util.util:272:enumerateWithEstimate Stuffing cache 32/539, done at 2019-08-18 13:51:16, 0:24:14\n", "2019-08-18 13:29:34,831 INFO pid:6276 util.util:272:enumerateWithEstimate Stuffing cache 64/539, done at 2019-08-18 13:50:46, 0:23:44\n", "2019-08-18 13:32:27,482 INFO pid:6276 util.util:272:enumerateWithEstimate Stuffing cache 128/539, done at 2019-08-18 13:50:50, 0:23:48\n", "2019-08-18 13:38:55,698 INFO pid:6276 util.util:272:enumerateWithEstimate Stuffing cache 256/539, done at 2019-08-18 13:52:24, 0:25:22\n", "2019-08-18 13:53:20,526 INFO pid:6276 util.util:272:enumerateWithEstimate Stuffing cache 512/539, done at 2019-08-18 13:54:41, 0:27:39\n", "2019-08-18 13:54:31,460 WARNING pid:6276 util.util:285:enumerateWithEstimate Stuffing cache ----/539, done at 2019-08-18 13:54:31\n", "2019-08-18 13:54:31,472 INFO pid:6276 nb:013:run Finished: p2ch10.prepcache.LunaPrepCacheApp.None).main()\n" ] } ], "source": [ "run('p2ch11.prepcache.LunaPrepCacheApp')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "2019-08-18 14:03:26,689 INFO pid:6276 nb:008:run Running: p2ch10.training.LunaTrainingApp(['--epochs=1']).main()\n", "2019-08-18 14:03:29,660 INFO pid:6276 p2ch10.training:155:main Starting LunaTrainingApp, Namespace(batch_size=32, comment='none', epochs=1, num_workers=8, tb_prefix='p2ch10')\n", "2019-08-18 14:03:30,029 INFO pid:6276 p2ch10.dsets:165:__init__ : 495958 training samples\n", "2019-08-18 14:03:30,079 INFO pid:6276 p2ch10.dsets:165:__init__ : 55107 validation samples\n", "2019-08-18 14:03:30,088 INFO pid:6276 p2ch10.training:181:main Epoch 1 of 1, 15499/1723 batches of size 32*1\n", "2019-08-18 14:03:30,093 WARNING pid:6276 util.util:248:enumerateWithEstimate E1 Training ----/15499, starting\n", "2019-08-18 14:03:55,834 INFO pid:6276 util.util:272:enumerateWithEstimate E1 Training 16/15499, done at 2019-08-18 14:19:48, 0:15:53\n", "2019-08-18 14:03:56,800 INFO pid:6276 util.util:272:enumerateWithEstimate E1 Training 32/15499, done at 2019-08-18 14:19:37, 0:15:42\n", "2019-08-18 14:03:58,747 INFO pid:6276 util.util:272:enumerateWithEstimate E1 Training 64/15499, done at 2019-08-18 14:19:37, 0:15:42\n", "2019-08-18 14:04:02,615 INFO pid:6276 util.util:272:enumerateWithEstimate E1 Training 128/15499, done at 2019-08-18 14:19:34, 0:15:38\n", "2019-08-18 14:04:10,374 INFO pid:6276 util.util:272:enumerateWithEstimate E1 Training 256/15499, done at 2019-08-18 14:19:34, 0:15:39\n", "2019-08-18 14:04:25,850 INFO pid:6276 util.util:272:enumerateWithEstimate E1 Training 512/15499, done at 2019-08-18 14:19:33, 0:15:37\n", "2019-08-18 14:04:56,935 INFO pid:6276 util.util:272:enumerateWithEstimate E1 Training 1024/15499, done at 2019-08-18 14:19:34, 0:15:39\n", "2019-08-18 14:05:59,204 INFO pid:6276 util.util:272:enumerateWithEstimate E1 Training 2048/15499, done at 2019-08-18 14:19:35, 0:15:40\n", "2019-08-18 14:08:03,762 INFO pid:6276 util.util:272:enumerateWithEstimate E1 Training 4096/15499, done at 2019-08-18 14:19:36, 0:15:41\n", "2019-08-18 14:12:14,139 INFO pid:6276 util.util:272:enumerateWithEstimate E1 Training 8192/15499, done at 2019-08-18 14:19:39, 0:15:44\n", "2019-08-18 14:19:41,875 WARNING pid:6276 util.util:285:enumerateWithEstimate E1 Training ----/15499, done at 2019-08-18 14:19:41\n", "2019-08-18 14:19:41,880 INFO pid:6276 p2ch10.training:291:logMetrics E1 LunaTrainingApp\n", "2019-08-18 14:19:41,900 INFO pid:6276 p2ch10.training:344:logMetrics E1 trn 0.0192 loss, 99.8% correct, \n", "2019-08-18 14:19:41,901 INFO pid:6276 p2ch10.training:356:logMetrics E1 trn_ben 0.0026 loss, 100.0% correct (494735 of 494743)\n", "2019-08-18 14:19:41,901 INFO pid:6276 p2ch10.training:369:logMetrics E1 trn_mal 6.7756 loss, 0.0% correct (0 of 1215)\n", "2019-08-18 14:19:41,975 WARNING pid:6276 util.util:248:enumerateWithEstimate E1 Validation ----/1723, starting\n", "2019-08-18 14:19:52,708 INFO pid:6276 util.util:272:enumerateWithEstimate E1 Validation 16/1723, done at 2019-08-18 14:21:15, 0:01:23\n", "2019-08-18 14:19:53,160 INFO pid:6276 util.util:272:enumerateWithEstimate E1 Validation 32/1723, done at 2019-08-18 14:20:53, 0:01:00\n", "2019-08-18 14:19:54,134 INFO pid:6276 util.util:272:enumerateWithEstimate E1 Validation 64/1723, done at 2019-08-18 14:20:48, 0:00:56\n", "2019-08-18 14:19:56,367 INFO pid:6276 util.util:272:enumerateWithEstimate E1 Validation 128/1723, done at 2019-08-18 14:20:50, 0:00:58\n", "2019-08-18 14:20:00,518 INFO pid:6276 util.util:272:enumerateWithEstimate E1 Validation 256/1723, done at 2019-08-18 14:20:49, 0:00:56\n", "2019-08-18 14:20:08,278 INFO pid:6276 util.util:272:enumerateWithEstimate E1 Validation 512/1723, done at 2019-08-18 14:20:46, 0:00:54\n", "2019-08-18 14:20:24,934 INFO pid:6276 util.util:272:enumerateWithEstimate E1 Validation 1024/1723, done at 2019-08-18 14:20:47, 0:00:55\n", "2019-08-18 14:20:46,957 WARNING pid:6276 util.util:285:enumerateWithEstimate E1 Validation ----/1723, done at 2019-08-18 14:20:46\n", "2019-08-18 14:20:46,959 INFO pid:6276 p2ch10.training:291:logMetrics E1 LunaTrainingApp\n", "2019-08-18 14:20:46,961 INFO pid:6276 p2ch10.training:344:logMetrics E1 val 0.0176 loss, 99.8% correct, \n", "2019-08-18 14:20:46,961 INFO pid:6276 p2ch10.training:356:logMetrics E1 val_ben 0.0014 loss, 100.0% correct (54971 of 54971)\n", "2019-08-18 14:20:46,962 INFO pid:6276 p2ch10.training:369:logMetrics E1 val_mal 6.5590 loss, 0.0% correct (0 of 136)\n" ] } ], "source": [ "run('p2ch11.training.LunaTrainingApp', '--epochs=1')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('p2ch11.training.LunaTrainingApp', f'--epochs={experiment_epochs}')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Chapter 12" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('p2ch12.prepcache.LunaPrepCacheApp')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('p2ch12.training.LunaTrainingApp', '--epochs=1', 'unbalanced')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('p2ch12.training.LunaTrainingApp', f'--epochs={training_epochs}', '--balanced', 'balanced')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('p2ch12.training.LunaTrainingApp', f'--epochs={experiment_epochs}', '--balanced', '--augment-flip', 'flip')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('p2ch12.training.LunaTrainingApp', f'--epochs={experiment_epochs}', '--balanced', '--augment-offset', 'offset')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('p2ch12.training.LunaTrainingApp', f'--epochs={experiment_epochs}', '--balanced', '--augment-scale', 'scale')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('p2ch12.training.LunaTrainingApp', f'--epochs={experiment_epochs}', '--balanced', '--augment-rotate', 'rotate')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('p2ch12.training.LunaTrainingApp', f'--epochs={experiment_epochs}', '--balanced', '--augment-noise', 'noise')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('p2ch12.training.LunaTrainingApp', f'--epochs={training_epochs}', '--balanced', '--augmented', 'fully-augmented')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Chapter 13" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('p2ch13.prepcache.LunaPrepCacheApp')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('p2ch13.train_cls.LunaTrainingApp', f'--epochs={final_epochs}', '--balanced', '--augmented', 'final-cls')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('p2ch13.train_seg.LunaTrainingApp', f'--epochs={training_epochs}', '--balanced', '--augmented', 'final-seg')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Chapter 14" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('p2ch14.diagnose.LunaDiagnoseApp')" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.6" } }, "nbformat": 4, "nbformat_minor": 2 }