{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Model comparison with the AIC\n", "\n", "**[mRNA count dataset download](https://s3.amazonaws.com/bebi103.caltech.edu/data/singer_transcript_counts.csv)**\n", "\n", "**[Spindle length dataset download](https://s3.amazonaws.com/bebi103.caltech.edu/data/good_invitro_droplet_data.csv)**\n", "\n", "
" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "
\n", " \n", " Loading BokehJS ...\n", "
" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/javascript": [ "\n", "(function(root) {\n", " function now() {\n", " return new Date();\n", " }\n", "\n", " var force = true;\n", "\n", " if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n", " root._bokeh_onload_callbacks = [];\n", " root._bokeh_is_loading = undefined;\n", " }\n", "\n", " var JS_MIME_TYPE = 'application/javascript';\n", " var HTML_MIME_TYPE = 'text/html';\n", " var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n", " var CLASS_NAME = 'output_bokeh rendered_html';\n", "\n", " /**\n", " * Render data to the DOM node\n", " */\n", " function render(props, node) {\n", " var script = document.createElement(\"script\");\n", " node.appendChild(script);\n", " }\n", "\n", " /**\n", " * Handle when an output is cleared or removed\n", " */\n", " function handleClearOutput(event, handle) {\n", " var cell = handle.cell;\n", "\n", " var id = cell.output_area._bokeh_element_id;\n", " var server_id = cell.output_area._bokeh_server_id;\n", " // Clean up Bokeh references\n", " if (id != null && id in Bokeh.index) {\n", " Bokeh.index[id].model.document.clear();\n", " delete Bokeh.index[id];\n", " }\n", "\n", " if (server_id !== undefined) {\n", " // Clean up Bokeh references\n", " var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n", " cell.notebook.kernel.execute(cmd, {\n", " iopub: {\n", " output: function(msg) {\n", " var id = msg.content.text.trim();\n", " if (id in Bokeh.index) {\n", " Bokeh.index[id].model.document.clear();\n", " delete Bokeh.index[id];\n", " }\n", " }\n", " }\n", " });\n", " // Destroy server and session\n", " var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n", " cell.notebook.kernel.execute(cmd);\n", " }\n", " }\n", "\n", " /**\n", " * Handle when a new output is added\n", " */\n", " function handleAddOutput(event, handle) {\n", " var output_area = handle.output_area;\n", " var output = handle.output;\n", "\n", " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n", " if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n", " return\n", " }\n", "\n", " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", "\n", " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n", " toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n", " // store reference to embed id on output_area\n", " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", " }\n", " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", " var bk_div = document.createElement(\"div\");\n", " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", " var script_attrs = bk_div.children[0].attributes;\n", " for (var i = 0; i < script_attrs.length; i++) {\n", " toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n", " }\n", " // store reference to server id on output_area\n", " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", " }\n", " }\n", "\n", " function register_renderer(events, OutputArea) {\n", "\n", " function append_mime(data, metadata, element) {\n", " // create a DOM node to render to\n", " var toinsert = this.create_output_subarea(\n", " metadata,\n", " CLASS_NAME,\n", " EXEC_MIME_TYPE\n", " );\n", " this.keyboard_manager.register_events(toinsert);\n", " // Render to node\n", " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", " render(props, toinsert[toinsert.length - 1]);\n", " element.append(toinsert);\n", " return toinsert\n", " }\n", "\n", " /* Handle when an output is cleared or removed */\n", " events.on('clear_output.CodeCell', handleClearOutput);\n", " events.on('delete.Cell', handleClearOutput);\n", "\n", " /* Handle when a new output is added */\n", " events.on('output_added.OutputArea', handleAddOutput);\n", "\n", " /**\n", " * Register the mime type and append_mime function with output_area\n", " */\n", " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", " /* Is output safe? */\n", " safe: true,\n", " /* Index of renderer in `output_area.display_order` */\n", " index: 0\n", " });\n", " }\n", "\n", " // register the mime type if in Jupyter Notebook environment and previously unregistered\n", " if (root.Jupyter !== undefined) {\n", " var events = require('base/js/events');\n", " var OutputArea = require('notebook/js/outputarea').OutputArea;\n", "\n", " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", " register_renderer(events, OutputArea);\n", " }\n", " }\n", "\n", " \n", " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n", " root._bokeh_timeout = Date.now() + 5000;\n", " root._bokeh_failed_load = false;\n", " }\n", "\n", " var NB_LOAD_WARNING = {'data': {'text/html':\n", " \"
\\n\"+\n", " \"

\\n\"+\n", " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n", " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n", " \"

\\n\"+\n", " \"\\n\"+\n", " \"\\n\"+\n", " \"from bokeh.resources import INLINE\\n\"+\n", " \"output_notebook(resources=INLINE)\\n\"+\n", " \"\\n\"+\n", " \"
\"}};\n", "\n", " function display_loaded() {\n", " var el = document.getElementById(\"1001\");\n", " if (el != null) {\n", " el.textContent = \"BokehJS is loading...\";\n", " }\n", " if (root.Bokeh !== undefined) {\n", " if (el != null) {\n", " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n", " }\n", " } else if (Date.now() < root._bokeh_timeout) {\n", " setTimeout(display_loaded, 100)\n", " }\n", " }\n", "\n", "\n", " function run_callbacks() {\n", " try {\n", " root._bokeh_onload_callbacks.forEach(function(callback) {\n", " if (callback != null)\n", " callback();\n", " });\n", " } finally {\n", " delete root._bokeh_onload_callbacks\n", " }\n", " console.debug(\"Bokeh: all callbacks have finished\");\n", " }\n", "\n", " function load_libs(css_urls, js_urls, callback) {\n", " if (css_urls == null) css_urls = [];\n", " if (js_urls == null) js_urls = [];\n", "\n", " root._bokeh_onload_callbacks.push(callback);\n", " if (root._bokeh_is_loading > 0) {\n", " console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", " return null;\n", " }\n", " if (js_urls == null || js_urls.length === 0) {\n", " run_callbacks();\n", " return null;\n", " }\n", " console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", " root._bokeh_is_loading = css_urls.length + js_urls.length;\n", "\n", " function on_load() {\n", " root._bokeh_is_loading--;\n", " if (root._bokeh_is_loading === 0) {\n", " console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n", " run_callbacks()\n", " }\n", " }\n", "\n", " function on_error() {\n", " console.error(\"failed to load \" + url);\n", " }\n", "\n", " for (var i = 0; i < css_urls.length; i++) {\n", " var url = css_urls[i];\n", " const element = document.createElement(\"link\");\n", " element.onload = on_load;\n", " element.onerror = on_error;\n", " element.rel = \"stylesheet\";\n", " element.type = \"text/css\";\n", " element.href = url;\n", " console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n", " document.body.appendChild(element);\n", " }\n", "\n", " for (var i = 0; i < js_urls.length; i++) {\n", " var url = js_urls[i];\n", " var element = document.createElement('script');\n", " element.onload = on_load;\n", " element.onerror = on_error;\n", " element.async = false;\n", " element.src = url;\n", " console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", " document.head.appendChild(element);\n", " }\n", " };var element = document.getElementById(\"1001\");\n", " if (element == null) {\n", " console.error(\"Bokeh: ERROR: autoload.js configured with elementid '1001' but no matching script tag was found. \")\n", " return false;\n", " }\n", "\n", " function inject_raw_css(css) {\n", " const element = document.createElement(\"style\");\n", " element.appendChild(document.createTextNode(css));\n", " document.body.appendChild(element);\n", " }\n", "\n", " \n", " var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n", " var css_urls = [];\n", " \n", "\n", " var inline_js = [\n", " function(Bokeh) {\n", " Bokeh.set_log_level(\"info\");\n", " },\n", " function(Bokeh) {\n", " \n", " \n", " }\n", " ];\n", "\n", " function run_inline_js() {\n", " \n", " if (root.Bokeh !== undefined || force === true) {\n", " \n", " for (var i = 0; i < inline_js.length; i++) {\n", " inline_js[i].call(root, root.Bokeh);\n", " }\n", " if (force === true) {\n", " display_loaded();\n", " }} else if (Date.now() < root._bokeh_timeout) {\n", " setTimeout(run_inline_js, 100);\n", " } else if (!root._bokeh_failed_load) {\n", " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", " root._bokeh_failed_load = true;\n", " } else if (force !== true) {\n", " var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n", " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n", " }\n", "\n", " }\n", "\n", " if (root._bokeh_is_loading === 0) {\n", " console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", " run_inline_js();\n", " } else {\n", " load_libs(css_urls, js_urls, function() {\n", " console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n", " run_inline_js();\n", " });\n", " }\n", "}(window));" ], "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n \n\n \n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n var NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded() {\n var el = document.getElementById(\"1001\");\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error() {\n console.error(\"failed to load \" + url);\n }\n\n for (var i = 0; i < css_urls.length; i++) {\n var url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error;\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n var element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error;\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };var element = document.getElementById(\"1001\");\n if (element == null) {\n console.error(\"Bokeh: ERROR: autoload.js configured with elementid '1001' but no matching script tag was found. \")\n return false;\n }\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n \n var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n var css_urls = [];\n \n\n var inline_js = [\n function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\n function(Bokeh) {\n \n \n }\n ];\n\n function run_inline_js() {\n \n if (root.Bokeh !== undefined || force === true) {\n \n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n if (force === true) {\n display_loaded();\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));" }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import warnings\n", "\n", "import pandas as pd\n", "import numpy as np\n", "import numba\n", "import scipy.optimize\n", "import scipy.stats as st\n", "\n", "import bebi103\n", "\n", "import bokeh_catplot\n", "\n", "import bokeh.io\n", "bokeh.io.output_notebook()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "
\n", "\n", "We have seen that we can assess models graphically, and even compare one model to another. In the example of the counts of Rex1 transcripts, we compared the model of a single Negative Binomial distribution to that of a mixture of two Negative Binomials. In the example of spindle length, we compared a model where the spindle length is independent of droplet diameter to one where it depends on droplet diameter according to conservation of tubulin.\n", "\n", "There are many non-graphical ways to assess models, including **[likelihood-ratio tests](https://en.wikipedia.org/wiki/Likelihood-ratio_test)** and **[cross-validation](https://en.wikipedia.org/wiki/Cross-validation_(statistics)**. Both of these are involved topics (especially cross-validation; there is a *lot* to learn there), and we will not cover them in much depth here. We will go into much more depth on model selection next term." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## The Akaike information criterion\n", "\n", "For now, we will consider a metric for model selection called the **[Akaike information criterion](https://en.wikipedia.org/wiki/Akaike_information_criterion)**, or AIC. We will not show it, but the AIC can be loosely interpreted as an estimate of a a quantity related to the distance between the true generative distribution and the model distribution. (I am speaking loosely here, there are important details missing, and in fact I have abused some language.) Given two models, the one with the lesser AIC is likely closer to the true generative distribution.\n", "\n", "For a set of parameters $\\theta$ with MLE $\\theta^*$ and a model with log-likelihood $\\ell(\\theta;\\text{data})$, the AIC is given by\n", "\n", "\\begin{align}\n", "\\text{AIC} = -2\\ell(\\theta^*;\\text{data}) + 2p,\n", "\\end{align}\n", "\n", "where $p$ is the number of free parameters in a model. Thus, more complicated models, those with more parameters, are penalized for having too many parameters. Furthermore, models where the likelihood function is broad, meaning that many parameter values are viable, have a lower log likelihood and are also penalized. So, models with few, meaningful parameters have lower AIC and are favored." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Akaike weights\n", "If we compare two models, say model 1 and model 2, then the **Akaike weight** for model 1 is\n", "\n", "\\begin{align}\n", "w_1 = \\frac{\\mathrm{e}^{-\\text{AIC}_1/2}}{\\mathrm{e}^{-\\text{AIC}_1/2} + \\mathrm{e}^{-\\text{AIC}_2/2}}.\n", "\\end{align}\n", "\n", "Because the values of the AIC can be large, it is more numerically stable to compute the Akaike weight as\n", "\n", "\\begin{align}\n", "w_1 = \\frac{\\mathrm{e}^{-(\\text{AIC}_1 - \\text{AIC}_\\mathrm{max})/2}}{\\mathrm{e}^{-(\\text{AIC}_1 - \\text{AIC}_\\mathrm{max})/2}+\\mathrm{e}^{-(\\text{AIC}_2 - \\text{AIC}_\\mathrm{max})/2}},\n", "\\end{align}\n", "\n", "where $\\text{AIC}_\\mathrm{max}$ is the maximum of the two AICs. More generally, the Akaike weight of model $i$ among many is\n", "\n", "\\begin{align}\n", "w_i = \\frac{\\mathrm{e}^{-(\\text{AIC}_i - \\text{AIC}_\\mathrm{max})/2}}{\\sum_j\\mathrm{e}^{-(\\text{AIC}_j - \\text{AIC}_\\mathrm{max})/2}}.\n", "\\end{align}\n", "\n", "Interpretation of the Akaike weights is lacking a consensus, but we can loosely interpret the Akaike weight as the probability that a model will give the best predictions of new data, assuming that we have considered all possible models." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## AIC for mRNA counts\n", "\n", "Let us now compare the single Negative Binomial to the mixture model for mRNA counts. We again need our functions for computing the MLE and computing the log-likelihood from previous lessons." ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "def log_like_iid_nbinom(params, n):\n", " \"\"\"Log likelihood for i.i.d. NBinom measurements, parametrized\n", " by alpha, b=1/beta.\"\"\"\n", " alpha, b = params\n", "\n", " if alpha <= 0 or b <= 0:\n", " return -np.inf\n", "\n", " return np.sum(st.nbinom.logpmf(n, alpha, 1/(1+b)))\n", "\n", "\n", "def mle_iid_nbinom(n):\n", " \"\"\"Perform maximum likelihood estimates for parameters for i.i.d.\n", " NBinom measurements, parametrized by alpha, b=1/beta\"\"\"\n", " with warnings.catch_warnings():\n", " warnings.simplefilter(\"ignore\")\n", "\n", " res = scipy.optimize.minimize(\n", " fun=lambda params, n: -log_like_iid_nbinom(params, n),\n", " x0=np.array([3, 3]),\n", " args=(n,),\n", " method='Powell'\n", " )\n", "\n", " if res.success:\n", " return res.x\n", " else:\n", " raise RuntimeError('Convergence failed with message', res.message)\n", " \n", "\n", "def initial_guess_mix(n, w_guess):\n", " \"\"\"Generate initial guess for mixture model.\"\"\"\n", " n_low = n[n < np.percentile(n, 100*w_guess)]\n", " n_high = n[n >= np.percentile(n, 100*w_guess)]\n", " \n", " alpha1, b1 = mle_iid_nbinom(n_low)\n", " alpha2, b2 = mle_iid_nbinom(n_high)\n", " \n", " return alpha1, b1, alpha2, b2\n", "\n", "\n", "def log_like_mix(alpha1, b1, alpha2, b2, w, n):\n", " \"\"\"Log-likeihood of binary Negative Binomial mixture model.\"\"\"\n", " # Fix nonidentifieability be enforcing values of w\n", " if w < 0 or w > 1:\n", " return -np.inf\n", " \n", " # Physical bounds on parameters\n", " if alpha1 < 0 or alpha2 < 0 or b1 < 0 or b2 < 0:\n", " return -np.inf\n", "\n", " logx1 = st.nbinom.logpmf(n, alpha1, 1/(1+b1))\n", " logx2 = st.nbinom.logpmf(n, alpha2, 1/(1+b2))\n", "\n", " # Multipliers for log-sum-exp\n", " lse_coeffs = np.tile([w, 1-w], [len(n), 1]).transpose()\n", "\n", " # log-likelihood for each measurement\n", " log_likes = scipy.special.logsumexp(np.vstack([logx1, logx2]), axis=0, b=lse_coeffs)\n", " \n", " return np.sum(log_likes)\n", "\n", "\n", "def mle_mix(n, w_guess):\n", " \"\"\"Obtain MLE estimate for parameters for binary mixture \n", " of Negative Binomials.\"\"\"\n", " with warnings.catch_warnings():\n", " warnings.simplefilter(\"ignore\")\n", "\n", " res = scipy.optimize.minimize(\n", " fun=lambda params, n: -log_like_mix(*params, n),\n", " x0=[*initial_guess_mix(n, w_guess), w_guess],\n", " args=(n,),\n", " method='Powell',\n", " tol=1e-6,\n", " )\n", "\n", " if res.success:\n", " return res.x\n", " else:\n", " raise RuntimeError('Convergence failed with message', res.message) " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now we can load in the data and compute the MLEs for each of the four genes." ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Rex1RestNanogPrdm14
alpha1.6345624.5303351.2630970.552886
b84.68091516.54305469.3478428.200636
alpha13.4970132.7866000.8348432.274491
b14.10491112.39570766.5355744.823948
alpha25.0896236.6834254.1275650.560992
b231.81038511.95326428.1326374.541668
w0.1604220.1087720.4666460.235637
\n", "
" ], "text/plain": [ " Rex1 Rest Nanog Prdm14\n", "alpha 1.634562 4.530335 1.263097 0.552886\n", "b 84.680915 16.543054 69.347842 8.200636\n", "alpha1 3.497013 2.786600 0.834843 2.274491\n", "b1 4.104911 12.395707 66.535574 4.823948\n", "alpha2 5.089623 6.683425 4.127565 0.560992\n", "b2 31.810385 11.953264 28.132637 4.541668\n", "w 0.160422 0.108772 0.466646 0.235637" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Load in data\n", "df = pd.read_csv(\"../data/singer_transcript_counts.csv\", comment=\"#\")\n", "\n", "df_mle = pd.DataFrame(index=['alpha', 'b', 'alpha1', 'b1', 'alpha2', 'b2', 'w'])\n", "\n", "for gene in df:\n", " n = df['Nanog'].values\n", "\n", " # Single Negative Binomial MLE\n", " alpha, b = mle_iid_nbinom(df[gene].values)\n", "\n", " # Mixture model MLE\n", " alpha1, b1, alpha2, b2, w = mle_mix(df[gene].values, 0.2)\n", " \n", " # Store results in data frame\n", " df_mle[gene] = [alpha, b, alpha1, b1, alpha2, b2, w]\n", " \n", "# Take a look\n", "df_mle" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "For each of the two models, we can compute the log likelihood evaluated at the MLEs for the parameters." ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Rex1RestNanogPrdm14
alpha1.6345624.5303351.2630970.552886
b84.68091516.54305469.3478428.200636
alpha13.4970132.7866000.8348432.274491
b14.10491112.39570766.5355744.823948
alpha25.0896236.6834254.1275650.560992
b231.81038511.95326428.1326374.541668
w0.1604220.1087720.4666460.235637
log_like_single-1638.678482-1376.748398-1524.928918-713.091587
log_like_mix-1590.353743-1372.108896-1512.444552-712.704687
\n", "
" ], "text/plain": [ " Rex1 Rest Nanog Prdm14\n", "alpha 1.634562 4.530335 1.263097 0.552886\n", "b 84.680915 16.543054 69.347842 8.200636\n", "alpha1 3.497013 2.786600 0.834843 2.274491\n", "b1 4.104911 12.395707 66.535574 4.823948\n", "alpha2 5.089623 6.683425 4.127565 0.560992\n", "b2 31.810385 11.953264 28.132637 4.541668\n", "w 0.160422 0.108772 0.466646 0.235637\n", "log_like_single -1638.678482 -1376.748398 -1524.928918 -713.091587\n", "log_like_mix -1590.353743 -1372.108896 -1512.444552 -712.704687" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "for gene in df:\n", " ell = log_like_iid_nbinom(\n", " (df_mle.loc[\"alpha\", gene], df_mle.loc[\"b\", gene]), df[gene].values\n", " )\n", " df_mle.loc[\"log_like_single\", gene] = ell\n", "\n", " ell_mix = log_like_mix(\n", " *df_mle.loc[[\"alpha1\", 'b1', 'alpha2', 'b2', 'w'], gene].values,\n", " df[gene].values,\n", " )\n", " df_mle.loc[\"log_like_mix\", gene] = ell_mix\n", " \n", "# Take a look\n", "df_mle" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We can already see a very large difference between the log likelihood evaluated at the MLE for Rex1, but not much difference for Prdm14. The mixture morel has $p = 5$ parameters, while the single Negative Binomial model has $p = 2$. With these numbers, we can compute the AIC and then also the Akaike weights." ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Rex1RestNanogPrdm14
alpha1.6345624.5303351.2630970.552886
b84.68091516.54305469.3478428.200636
alpha13.4970132.7866000.8348432.274491
b14.10491112.39570766.5355744.823948
alpha25.0896236.6834254.1275650.560992
b231.81038511.95326428.1326374.541668
w0.1604220.1087720.4666460.235637
log_like_single-1638.678482-1376.748398-1524.928918-713.091587
log_like_mix-1590.353743-1372.108896-1512.444552-712.704687
AIC_single3281.3569632757.4967973053.8578371430.183173
AIC_mix3190.7074872754.2177923034.8891031435.409375
\n", "
" ], "text/plain": [ " Rex1 Rest Nanog Prdm14\n", "alpha 1.634562 4.530335 1.263097 0.552886\n", "b 84.680915 16.543054 69.347842 8.200636\n", "alpha1 3.497013 2.786600 0.834843 2.274491\n", "b1 4.104911 12.395707 66.535574 4.823948\n", "alpha2 5.089623 6.683425 4.127565 0.560992\n", "b2 31.810385 11.953264 28.132637 4.541668\n", "w 0.160422 0.108772 0.466646 0.235637\n", "log_like_single -1638.678482 -1376.748398 -1524.928918 -713.091587\n", "log_like_mix -1590.353743 -1372.108896 -1512.444552 -712.704687\n", "AIC_single 3281.356963 2757.496797 3053.857837 1430.183173\n", "AIC_mix 3190.707487 2754.217792 3034.889103 1435.409375" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "for gene in df:\n", " df_mle.loc['AIC_single', gene] = -2 * (df_mle.loc['log_like_single', gene] - 2)\n", " df_mle.loc['AIC_mix', gene] = -2 * (df_mle.loc['log_like_mix', gene] - 5)\n", " \n", "# Take a look\n", "df_mle" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Finally, we can compute the Akaike weight for the mixture model (the weight for the single Negative Binomial model is $1-w_\\mathrm{mix}$." ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Rex1RestNanogPrdm14
alpha1.634562e+004.5303351.2630970.552886
b8.468092e+0116.54305469.3478428.200636
alpha13.497013e+002.7866000.8348432.274491
b14.104911e+0012.39570766.5355744.823948
alpha25.089623e+006.6834254.1275650.560992
b23.181038e+0111.95326428.1326374.541668
w1.604216e-010.1087720.4666460.235637
log_like_single-1.638678e+03-1376.748398-1524.928918-713.091587
log_like_mix-1.590354e+03-1372.108896-1512.444552-712.704687
AIC_single3.281357e+032757.4967973053.8578371430.183173
AIC_mix3.190707e+032754.2177923034.8891031435.409375
w_single2.068789e-200.1625330.0000760.931700
\n", "
" ], "text/plain": [ " Rex1 Rest Nanog Prdm14\n", "alpha 1.634562e+00 4.530335 1.263097 0.552886\n", "b 8.468092e+01 16.543054 69.347842 8.200636\n", "alpha1 3.497013e+00 2.786600 0.834843 2.274491\n", "b1 4.104911e+00 12.395707 66.535574 4.823948\n", "alpha2 5.089623e+00 6.683425 4.127565 0.560992\n", "b2 3.181038e+01 11.953264 28.132637 4.541668\n", "w 1.604216e-01 0.108772 0.466646 0.235637\n", "log_like_single -1.638678e+03 -1376.748398 -1524.928918 -713.091587\n", "log_like_mix -1.590354e+03 -1372.108896 -1512.444552 -712.704687\n", "AIC_single 3.281357e+03 2757.496797 3053.857837 1430.183173\n", "AIC_mix 3.190707e+03 2754.217792 3034.889103 1435.409375\n", "w_single 2.068789e-20 0.162533 0.000076 0.931700" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "for gene in df:\n", " AIC_max = max(df_mle.loc[['AIC_single', 'AIC_mix'], gene])\n", " numerator = np.exp(-(df_mle.loc['AIC_single', gene] - AIC_max)/2)\n", " denominator = numerator + np.exp(-(df_mle.loc['AIC_mix', gene] - AIC_max)/2)\n", " df_mle.loc['w_single', gene] = numerator / denominator\n", " \n", "# Take a look\n", "df_mle" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "In looking at the Akaike weight for the mixture (`w_mix`), is it clear that the mixture model is strongly preferred for Rex1 and Nanog. There is not strong preference for Rest, and a preference for the single Negative Binomial model for Prdm14. Reminding ourselves of the ECDFs, this makes sense." ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n", "\n", "\n", "\n", "\n", "
\n" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/javascript": [ "(function(root) {\n", " function embed_document(root) {\n", " \n", " var docs_json = {\"5826a1ec-19cf-4e0f-8a1a-0ac57003a5ee\":{\"roots\":{\"references\":[{\"attributes\":{\"children\":[{\"id\":\"1232\",\"type\":\"ToolbarBox\"},{\"id\":\"1230\",\"type\":\"GridBox\"}]},\"id\":\"1233\",\"type\":\"Column\"},{\"attributes\":{\"toolbar\":{\"id\":\"1231\",\"type\":\"ProxyToolbar\"},\"toolbar_location\":\"above\"},\"id\":\"1232\",\"type\":\"ToolbarBox\"},{\"attributes\":{\"items\":[{\"id\":\"1049\",\"type\":\"LegendItem\"}],\"visible\":false},\"id\":\"1048\",\"type\":\"Legend\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"x\":{\"field\":\"Nanog\"},\"y\":{\"field\":\"__ECDF\"}},\"id\":\"1039\",\"type\":\"Circle\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1115\",\"type\":\"BasicTicker\"}},\"id\":\"1118\",\"type\":\"Grid\"},{\"attributes\":{\"source\":{\"id\":\"1084\",\"type\":\"ColumnDataSource\"}},\"id\":\"1089\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1009\",\"type\":\"LinearScale\"},{\"attributes\":{\"callback\":null,\"data\":{\"Rest\":[34,91,70,54,54,68,74,60,96,35,50,51,43,50,74,125,62,130,95,31,67,90,62,39,68,40,65,78,60,94,99,55,69,41,102,65,42,76,114,84,106,182,68,86,63,24,103,96,75,60,70,79,74,74,44,104,97,30,58,17,34,44,28,49,65,246,104,108,91,144,40,80,57,82,85,51,39,98,88,65,13,140,85,76,21,80,60,137,91,70,12,53,65,33,59,59,72,110,138,105,83,104,62,57,110,51,106,92,59,33,108,53,34,94,48,64,93,53,77,83,56,48,88,54,75,46,29,42,79,64,57,92,125,37,44,112,52,125,152,18,34,16,53,72,103,35,75,62,134,83,94,67,84,80,57,80,29,87,130,32,142,78,92,74,155,159,61,74,94,59,91,112,68,27,32,78,162,89,162,92,100,40,37,5,107,31,91,153,94,129,98,108,8,53,95,39,65,68,75,83,45,90,87,78,117,36,82,61,129,103,55,43,40,56,27,87,53,72,89,109,124,49,31,48,82,70,82,121,65,11,170,103,62,46,106,94,82,95,91,64,58,66,26,67,79,118,26,109,25,54,51,53,52,12,70,62,116,85,101,92,134,74,46,73,30,137,107,59,46,86,15,62,82,148,77,91,92,82,98],\"__ECDF\":{\"__ndarray__\":\"hxtuuOGGuz/rAFm5OkDmP0geeeSRR94/ynxIdzIf0j+vZ0nQ61nSP+ecc84559w/LLDAAgss4D/Q61kS9HrWPzmO4ziO4+g/3cl8SHcyvz+ycnWArFzNP0geeeSRR84/0OtZEvR6xj99SHcyH9LNP58lQa9nSeA/P/30008/7T8xbV8oTNvXP31IdzIf0u0/4S1iM3iL6D9walT8mxq1PzfccMMNN9w/BhZYYIEF5j8WWGCBBRbYP+SRRx555ME/zYd0J/Mh3T9FE0000UTDPwyFaftCYdo/5JFHHnnk4T+21lprrbXWPzFtXyhM2+c/6U7mQ7qT6T9fKEzbFwrTP2IzeIvYDN4/cGpU/JsaxT9Br2dJ0OvpP/FvalT8m9o/O0BWrg6QxT8avEVsBm/hP4888sgjj+w/wKlR8W9q5D9J0OtZEvTqP42Kf1Oj4u8/snJ1gKxc3T/99NNPP/3kP3bZZZdddtk/BhZYYIEFpj+0JOj1LAnqP6wDZOXqAOk/T+ZDupP54D+bwVvEZvDWPy0Jej1Lgt4/r2dJ0OtZ4j8Rm8FbxGbgP4QQQgghhOA/Zpdddtllxz9/+umnn37qPx555JFHHuk/RRNNNNFEsz+Lf1Oj4t/UP4QQQgghhKA/HMdxHMdxvD8xbV8oTNvHP4QQQgghhLA/HMdxHMdxzD/XWmuttdbaPwAAAAAAAPA/8W9qVPyb6j+HG2644YbrP1522WWXXeY/hWn7QmHa7j8Q6U7mQ7rDPwfIytUBsuI/9dNPP/300z/SncyHdCfjP6WUUkoppeQ/EvR6lgS9zj+vZ0nQ61nCP5HuZD6kO+k/O0BWrg6Q5T+8RWwGbxHbPwYWWGCBBZY/oH766aef7j8YCtP2hcLkP4wxxhhjjOE/2r5QmLYvpD96PUuCXs/iP4CsXB0gK9c/SB555JFH7j/Q61kS9HrmPxL0epYEvd4/snJ1gKxcjT+EEEIIIYTQP6EwbV8oTNs/XMRm8BaxuT9VVVVVVVXVPztAVq4OkNU/w7R9oTBt3z/EZvAWsRnsPy0Jej1Lgu4/11prrbXW6j/1008//fTjP2Tl6gBZueo/+0Jh2r5Q2D/avlCYti/UPzfccMMNN+w/3cl8SHcyzz+8RWwGbxHrPw433HDDDec/ICtXB8jK1T/xb2pU/Ju6P/qQ7mQ+pOs/avtCYdq+0D+ycnWArFy9P6Pi39So+Oc/vEVsBm8Ryz9cxGbwFrHZP7733nvvvec/T+ZDupP50D//pkbFv6nhP2hJ0OtZEuQ/Kv5NjYp/0z+HG2644YbLP6211lprreU/lVJKKaWU0j/CW8Rm8BbhP5HuZD6kO8k/GrxFbAZvsT8GFlhggQXGPyLdyXxId+I/Qa9nSdDr2T/AqVHxb2rUP4CsXB0gK+c/snJ1gKxc7T9P5kO6k/nAP/tCYdq+UMg/qlHxb2pU7D+5OkBWrg7QPyXo9SwJeu0/alT8mxoV7z+vZ0nQ61miP0geeeSRR74/snJ1gKxcnT800UQTTTTRP6iffvrpp98/J5poookm6j+5OkBWrg7APzTRRBNNNOE/4S1iM3iL2D9iM3iL2AzuP9q+UJi2L+Q/FlhggQUW6D8cx3Ecx3HcPzIf0p3Mh+Q/7bLLLrvs4j+llFJKKaXUP18oTNsXCuM/r2dJ0OtZsj/j39So+DflP++999577+0/MW1fKEzbtz8S9HqWBL3uP1cHyMrVAeI/8yHdyXxI5z/3hcK0faHgP1A//fTTT+8/w7R9oTBt7z9ml1122WXXP2r7QmHavuA/ic3gLWIz6D8GFlhggQXWP0Nh2r5QmOY/HMdxHMdx7D+XXXbZZZfdP7JydYCsXK0/xhhjjDHGuD/KfEh3Mh/iPzUq/k2Niu8/ICtXB8jK5T+on3766afvP2aXXXbZZec/XMRm8Bax6T/avlCYti/EPxq8RWwGb8E/snJ1gKxcbT+hMG1fKEzrPwYWWGCBBbY/ttZaa6215j/dyXxIdzLvP/tCYdq+UOg/l1122WWX7T8EZOXqAFnpP2wGbxGbwes/snJ1gKxcfT8avEVsBm/RP1Oj4t/UqOg/ej1Lgl7Pwj+HG2644YbbP31IdzIf0t0/p0bFv6lR4T9NNNFEE03kP8YYY4wxxsg/eIvYDN4i5j9VVVVVVVXlPz3yyCOPPOI/dSfzId3J7D+EEEIIIYTAP0UTTTTRROM/S4Jez5Kg1z8K0/aFwrTtP5kP6U7mQ+o/RRNNNNFE0z+bwVvEZvDGP6WUUkoppcQ/EOlO5kO60z/dyXxIdzKvP8jK1QGycuU//6ZGxb+p0T+Nin9To+LfP5Og17Mk6OU/33vvvffe6z/Nh3Qn8yHtP+ecc84558w/m8FbxGbwtj9S8W9qVPzLP7iIzeAtYuM/+N5777333j8q/k2Nin/jP1oS9HqWBO0/bAZvEZvB2z8GFlhggQWGPxsV/6ZGxe8/DIVp+0Jh6j/GGGOMMcbYP1zEZvAWsck/L7vssssu6z9uuOGGG27oP51zzjnnnOM/xhhjjDHG6D8oTNsXCtPmPyeaaKKJJto/cGpU/Jsa1T9S8W9qVPzbP1zEZvAWsak/ArJydYCs3D+VUkoppZTiP+ecc8455+w/hxtuuOGGqz9S8W9qVPzrPzFtXyhM26c/ej1Lgl7P0j+on3766afPP+SRRx555NE/nyVBr2dJ0D+vZ0nQ61mSP93JfEh3Mt8/rANk5eoA2T8CsnJ1gKzsP4t/U6Pi3+Q/zjnnnHPO6T/ZDN4iNoPnP9Wo+Dc1Ku4/3HDDDTfc4D8nmmiiiSbKP7k6QFauDuA/2r5QmLYvtD+6k/mQ7mTuPxSm7QuFaes/6wBZuTpA1j/xb2pU/JvKP3BqVPybGuU/XMRm8BaxmT+R7mQ+pDvZPxDpTuZDuuM/+N5777337j9yHMdxHMfhP5vBW8Rm8OY/S4Jez5Kg5z+CXs+SoNfjP3bZZZddduk/\",\"dtype\":\"float64\",\"shape\":[279]},\"__dummy_cat\":[\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \"],\"__label\":[\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \"],\"index\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278]},\"selected\":{\"id\":\"1219\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1218\",\"type\":\"UnionRenderers\"}},\"id\":\"1132\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"tools\":[{\"id\":\"1023\",\"type\":\"PanTool\"},{\"id\":\"1024\",\"type\":\"WheelZoomTool\"},{\"id\":\"1025\",\"type\":\"BoxZoomTool\"},{\"id\":\"1026\",\"type\":\"SaveTool\"},{\"id\":\"1027\",\"type\":\"ResetTool\"},{\"id\":\"1028\",\"type\":\"HelpTool\"},{\"id\":\"1071\",\"type\":\"PanTool\"},{\"id\":\"1072\",\"type\":\"WheelZoomTool\"},{\"id\":\"1073\",\"type\":\"BoxZoomTool\"},{\"id\":\"1074\",\"type\":\"SaveTool\"},{\"id\":\"1075\",\"type\":\"ResetTool\"},{\"id\":\"1076\",\"type\":\"HelpTool\"},{\"id\":\"1119\",\"type\":\"PanTool\"},{\"id\":\"1120\",\"type\":\"WheelZoomTool\"},{\"id\":\"1121\",\"type\":\"BoxZoomTool\"},{\"id\":\"1122\",\"type\":\"SaveTool\"},{\"id\":\"1123\",\"type\":\"ResetTool\"},{\"id\":\"1124\",\"type\":\"HelpTool\"},{\"id\":\"1167\",\"type\":\"PanTool\"},{\"id\":\"1168\",\"type\":\"WheelZoomTool\"},{\"id\":\"1169\",\"type\":\"BoxZoomTool\"},{\"id\":\"1170\",\"type\":\"SaveTool\"},{\"id\":\"1171\",\"type\":\"ResetTool\"},{\"id\":\"1172\",\"type\":\"HelpTool\"}]},\"id\":\"1231\",\"type\":\"ProxyToolbar\"},{\"attributes\":{\"text\":\"Nanog\"},\"id\":\"1003\",\"type\":\"Title\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1119\",\"type\":\"PanTool\"},{\"id\":\"1120\",\"type\":\"WheelZoomTool\"},{\"id\":\"1121\",\"type\":\"BoxZoomTool\"},{\"id\":\"1122\",\"type\":\"SaveTool\"},{\"id\":\"1123\",\"type\":\"ResetTool\"},{\"id\":\"1124\",\"type\":\"HelpTool\"}]},\"id\":\"1125\",\"type\":\"Toolbar\"},{\"attributes\":{\"callback\":null},\"id\":\"1005\",\"type\":\"DataRange1d\"},{\"attributes\":{\"axis_label\":\"ECDF\",\"formatter\":{\"id\":\"1187\",\"type\":\"BasicTickFormatter\"},\"ticker\":{\"id\":\"1163\",\"type\":\"BasicTicker\"}},\"id\":\"1162\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"1119\",\"type\":\"PanTool\"},{\"attributes\":{},\"id\":\"1120\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"source\":{\"id\":\"1036\",\"type\":\"ColumnDataSource\"}},\"id\":\"1041\",\"type\":\"CDSView\"},{\"attributes\":{\"overlay\":{\"id\":\"1143\",\"type\":\"BoxAnnotation\"}},\"id\":\"1121\",\"type\":\"BoxZoomTool\"},{\"attributes\":{},\"id\":\"1122\",\"type\":\"SaveTool\"},{\"attributes\":{\"ticker\":{\"id\":\"1110\",\"type\":\"BasicTicker\"}},\"id\":\"1113\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1045\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"data_source\":{\"id\":\"1036\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1038\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1039\",\"type\":\"Circle\"},\"selection_glyph\":null,\"view\":{\"id\":\"1041\",\"type\":\"CDSView\"}},\"id\":\"1040\",\"type\":\"GlyphRenderer\"},{\"attributes\":{},\"id\":\"1123\",\"type\":\"ResetTool\"},{\"attributes\":{\"axis_label\":\"mRNA count\",\"formatter\":{\"id\":\"1045\",\"type\":\"BasicTickFormatter\"},\"ticker\":{\"id\":\"1014\",\"type\":\"BasicTicker\"}},\"id\":\"1013\",\"type\":\"LinearAxis\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1163\",\"type\":\"BasicTicker\"}},\"id\":\"1166\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1124\",\"type\":\"HelpTool\"},{\"attributes\":{},\"id\":\"1163\",\"type\":\"BasicTicker\"},{\"attributes\":{\"ticker\":{\"id\":\"1158\",\"type\":\"BasicTicker\"}},\"id\":\"1161\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1011\",\"type\":\"LinearScale\"},{\"attributes\":{\"data_source\":{\"id\":\"1132\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1134\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1135\",\"type\":\"Circle\"},\"selection_glyph\":null,\"view\":{\"id\":\"1137\",\"type\":\"CDSView\"}},\"id\":\"1136\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"label\":{\"value\":\" \"},\"renderers\":[{\"id\":\"1040\",\"type\":\"GlyphRenderer\"}]},\"id\":\"1049\",\"type\":\"LegendItem\"},{\"attributes\":{},\"id\":\"1219\",\"type\":\"Selection\"},{\"attributes\":{\"text\":\"Rex1\"},\"id\":\"1147\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"1210\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1023\",\"type\":\"PanTool\"},{\"attributes\":{\"callback\":null},\"id\":\"1149\",\"type\":\"DataRange1d\"},{\"attributes\":{},\"id\":\"1218\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"lightgrey\"},\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":{\"value\":1.0},\"line_color\":{\"value\":\"black\"},\"line_dash\":[4,4],\"line_width\":{\"value\":2},\"render_mode\":\"css\",\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"1143\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"x\":{\"field\":\"Rest\"},\"y\":{\"field\":\"__ECDF\"}},\"id\":\"1135\",\"type\":\"Circle\"},{\"attributes\":{\"fill_color\":{\"value\":\"#1f77b3\"},\"line_color\":{\"value\":\"#1f77b3\"},\"x\":{\"field\":\"Rest\"},\"y\":{\"field\":\"__ECDF\"}},\"id\":\"1134\",\"type\":\"Circle\"},{\"attributes\":{\"source\":{\"id\":\"1132\",\"type\":\"ColumnDataSource\"}},\"id\":\"1137\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1141\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1014\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"1139\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"items\":[{\"id\":\"1145\",\"type\":\"LegendItem\"}],\"visible\":false},\"id\":\"1144\",\"type\":\"Legend\"},{\"attributes\":{\"fill_color\":{\"value\":\"#1f77b3\"},\"line_color\":{\"value\":\"#1f77b3\"},\"x\":{\"field\":\"Nanog\"},\"y\":{\"field\":\"__ECDF\"}},\"id\":\"1038\",\"type\":\"Circle\"},{\"attributes\":{\"callback\":null},\"id\":\"1007\",\"type\":\"DataRange1d\"},{\"attributes\":{},\"id\":\"1167\",\"type\":\"PanTool\"},{\"attributes\":{},\"id\":\"1168\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"overlay\":{\"id\":\"1191\",\"type\":\"BoxAnnotation\"}},\"id\":\"1169\",\"type\":\"BoxZoomTool\"},{\"attributes\":{\"fill_color\":{\"value\":\"#1f77b3\"},\"line_color\":{\"value\":\"#1f77b3\"},\"x\":{\"field\":\"Prdm14\"},\"y\":{\"field\":\"__ECDF\"}},\"id\":\"1086\",\"type\":\"Circle\"},{\"attributes\":{},\"id\":\"1170\",\"type\":\"SaveTool\"},{\"attributes\":{\"callback\":null,\"data\":{\"Prdm14\":[0,5,0,1,0,0,0,2,1,3,1,1,3,14,16,11,1,15,0,0,0,1,0,0,7,0,4,6,0,12,4,0,7,17,1,8,2,0,0,1,4,24,31,3,3,0,1,1,5,0,3,1,7,1,5,1,7,1,1,0,0,0,0,3,9,32,3,15,4,4,0,1,6,3,3,1,2,19,8,1,0,11,3,6,0,12,0,12,5,9,2,7,21,0,5,0,1,2,3,1,1,5,0,0,13,0,4,2,3,0,13,27,0,2,1,1,0,0,3,6,5,2,3,7,2,0,0,6,0,0,1,25,12,7,1,6,0,4,3,1,0,1,20,10,7,0,0,0,5,0,12,0,2,1,0,0,7,3,1,0,8,0,15,5,4,6,0,2,1,0,12,4,6,0,3,0,9,8,12,7,13,1,0,1,11,3,11,27,6,32,0,6,0,0,5,0,2,3,2,1,2,2,6,6,1,1,3,0,0,3,4,7,6,3,0,22,0,18,1,5,0,1,1,18,3,0,4,15,3,0,0,0,18,1,0,1,16,12,1,8,5,0,0,17,4,8,0,0,0,8,1,1,10,0,5,0,0,0,15,0,8,5,6,5,1,0,1,0,1,9,0,6,5,19,6,1,4,4,0],\"__ECDF\":{\"__ndarray__\":\"snJ1gKxcbT/99NNPP/3kP7JydYCsXH0/XyhM2xcK0z8GFlhggQWGP7JydYCsXI0/r2dJ0OtZkj99SHcyH9LdP0UTTTTRRNM/94XCtH2h4D8q/k2Nin/TPxDpTuZDutM/avtCYdq+4D8//fTTTz/tP++999577+0/hxtuuOGG6z/1008//fTTP7JydYCsXO0/BhZYYIEFlj9cxGbwFrGZP7JydYCsXJ0/2r5QmLYv1D+EEEIIIYSgP69nSdDrWaI/U6Pi39So6D/avlCYti+kP7iIzeAtYuM/KEzbFwrT5j8GFlhggQWmP1Lxb2pU/Os/Kv5NjYp/4z8xbV8oTNunP8YYY4wxxug/1aj4NzUq7j/AqVHxb2rUP0GvZ0nQ6+k/YjN4i9gM3j9cxGbwFrGpP4cbbrjhhqs/pZRSSiml1D+dc84555zjP1A//fTTT+8/GxX/pkbF7z/ccMMNN9zgP0/mQ7qT+eA/snJ1gKxcrT+Lf1Oj4t/UP3BqVPybGtU/cGpU/Jsa5T/dyXxIdzKvP8JbxGbwFuE/VVVVVVVV1T85juM4juPoPztAVq4OkNU/49/UqPg35T8gK1cHyMrVP6wDZOXqAOk/BhZYYIEF1j/rAFm5OkDWP4QQQgghhLA/GrxFbAZvsT+vZ0nQ61myP0UTTTTRRLM/NNFEE0004T/XWmuttdbqP42Kf1Oj4u8/p0bFv6lR4T8l6PUsCXrtPxDpTuZDuuM/gl7PkqDX4z/avlCYti+0P9DrWRL0etY/m8FbxGbw5j8avEVsBm/hP4wxxhhjjOE/ttZaa6211j9IHnnkkUfePxL0epYEve4/tCTo9SwJ6j+bwVvEZvDWP3BqVPybGrU/+pDuZD6k6z//pkbFv6nhPw433HDDDec/BhZYYIEFtj/EZvAWsRnsP5vBW8Rm8LY/N9xwww037D9VVVVVVVXlP0nQ61kS9Oo/LQl6PUuC3j8eeeSRRx7pP2pU/JsaFe8/MW1fKEzbtz/IytUBsnLlP8YYY4wxxrg/gKxcHSAr1z8S9HqWBL3eP3Icx3Ecx+E/Zpdddtll1z9Lgl7PkqDXPztAVq4OkOU/XMRm8BaxuT/xb2pU/Ju6P+ecc8455+w/hxtuuOGGuz/1008//fTjP/jee++9994/5JFHHnnk4T8cx3Ecx3G8P1oS9HqWBO0/NSr+TY2K7z+ycnWArFy9P93JfEh3Mt8/MW1fKEzb1z8WWGCBBRbYP0geeeSRR74/3cl8SHcyvz9XB8jK1QHiP4CsXB0gK+c/rbXWWmut5T/DtH2hMG3fP8p8SHcyH+I/ke5kPqQ76T+on3766affP7k6QFauDsA/hBBCCCGEwD/zId3JfEjnP0/mQ7qT+cA/GrxFbAZvwT/7QmHavlDYP8O0faEwbe8/qlHxb2pU7D8EZOXqAFnpP+EtYjN4i9g/Zpdddtll5z/kkUceeeTBP2hJ0OtZEuQ/PfLII4884j/GGGOMMcbYP69nSdDrWcI/rANk5eoA2T/43nvvvffuP6EwbV8oTOs/dtlll1126T96PUuCXs/CP0UTTTTRRMM/EOlO5kO6wz8gK1cHyMrlP9q+UJi2L8Q/HMdxHMdx7D+llFJKKaXEP42Kf1Oj4t8/ke5kPqQ72T9walT8mxrFPztAVq4OkMU/6U7mQ7qT6T+vZ0nQ61niP3bZZZdddtk/BhZYYIEFxj8nmmiiiSbqP9DrWRL0esY/l1122WWX7T+ToNezJOjlP9q+UJi2L+Q/2QzeIjaD5z+bwVvEZvDGP7k6QFauDuA/XMRm8Bax2T9ml1122WXHP4888sgjj+w/TTTRRBNN5D9Lgl7PkqDnPzFtXyhM28c/It3JfEh34j/7QmHavlDIP7xFbAZvEes/mQ/pTuZD6j8CsnJ1gKzsP1zEZvAWsek/zYd0J/Mh7T9Br2dJ0OvZP8YYY4wxxsg/J5poookm2j9sBm8Rm8HrP5VSSimllOI/33vvvffe6z+on3766afvP7733nvvvec/AAAAAAAA8D+R7mQ+pDvJPzFtXyhM2+c/XMRm8BaxyT8nmmiiiSbKPwYWWGCBBeY/8W9qVPybyj8ssMACCyzgPwfIytUBsuI/nyVBr2dJ4D8MhWn7QmHaPxGbwVvEZuA/hBBCCCGE4D+j4t/UqPjnPxZYYIEFFug/8W9qVPyb2j/XWmuttdbaP3o9S4Jez+I/vEVsBm8Ryz+HG2644YbLP+2yyy677OI/wKlR8W9q5D/OOeecc87pP4nN4C1iM+g/XyhM2xcK4z9S8W9qVPzLP93JfEh3Mu8/HMdxHMdxzD+6k/mQ7mTuP7xFbAZvEds/eIvYDN4i5j/nnHPOOefMP6EwbV8oTNs/hxtuuOGG2z8tCXo9S4LuP9KdzId0J+M/snJ1gKxczT8yH9KdzIfkPwrT9oXCtO0/RRNNNNFE4z99SHcyH9LNP0geeeSRR84/EvR6lgS9zj+gfvrpp5/uP2wGbxGbwds/3cl8SHcyzz9S8W9qVPzbP2IzeIvYDO4/dSfzId3J7D833HDDDTfcPwyFaftCYeo/6wBZuTpA5j+on3766afPP7k6QFauDtA/SB555JFH7j+llFJKKaXkP3/66aeffuo/nyVBr2dJ0D+EEEIIIYTQP2r7QmHavtA/8W9qVPyb6j8cx3Ecx3HcPwKycnWArNw/FKbtC4Vp6z9P5kO6k/nQP1522WWXXeY/NNFEE0000T8avEVsBm/RP/+mRsW/qdE/fUh3Mh/S7T/kkUceeeTRP2Tl6gBZueo/0OtZEvR65j/7QmHavlDoP0Nh2r5QmOY/55xzzjnn3D/KfEh3Mh/SP82HdCfzId0/r2dJ0OtZ0j+ycnWArFzdPy+77LLLLus/lVJKKaWU0j9uuOGGG27oP7bWWmutteY/hWn7QmHa7j/hLWIzeIvoP5dddtlll90/GArT9oXC5D+Lf1Oj4t/kP3o9S4Jez9I/\",\"dtype\":\"float64\",\"shape\":[279]},\"__dummy_cat\":[\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \"],\"__label\":[\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \"],\"index\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278]},\"selected\":{\"id\":\"1210\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1209\",\"type\":\"UnionRenderers\"}},\"id\":\"1084\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1171\",\"type\":\"ResetTool\"},{\"attributes\":{},\"id\":\"1043\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"callback\":null,\"data\":{\"Rex1\":[11,172,261,178,129,97,16,29,118,133,152,8,125,172,245,265,202,291,14,40,56,160,120,52,239,11,151,76,122,290,98,170,218,171,131,89,32,8,214,219,172,353,331,235,209,8,297,239,192,238,124,163,148,151,40,205,331,218,150,61,76,128,11,166,96,362,160,247,204,266,98,145,61,136,111,15,80,250,169,155,6,163,174,227,69,221,167,174,155,195,38,135,244,98,182,10,191,131,255,288,137,238,103,131,166,17,284,264,86,8,260,81,19,121,9,126,8,6,110,194,118,122,122,159,23,12,163,192,302,157,62,206,104,140,129,126,160,140,258,30,99,6,98,170,254,9,11,154,145,11,90,203,186,144,68,166,94,186,201,113,176,18,156,31,173,166,108,245,191,113,72,157,51,147,108,11,420,190,151,153,198,154,48,11,95,23,134,326,124,105,233,217,3,94,144,47,177,241,54,102,142,204,89,81,124,7,265,100,204,102,116,142,210,140,29,155,6,90,96,142,281,118,5,116,271,19,321,226,201,8,17,179,289,25,262,227,194,98,130,116,42,79,15,193,276,143,93,242,32,153,111,53,47,6,252,78,205,217,215,13,191,116,132,125,63,186,97,100,54,137,4,158,166,180,195,144,67,120,280],\"__ECDF\":{\"__ndarray__\":\"GrxFbAZvsT87QFauDpDlP7JydYCsXO0/0OtZEvR65j+ycnWArFzdPyr+TY2Kf9M/HMdxHMdxvD+vZ0nQ61nCP3bZZZdddtk/+N5777333j898sgjjzziP69nSdDrWaI/N9xwww033D+ttdZaa63lP6pR8W9qVOw/CtP2hcK07T85juM4juPoP/jee++99+4/XMRm8BaxuT87QFauDpDFP/FvalT8m8o/EOlO5kO64z8nmmiiiSbaP8YYY4wxxsg/bAZvEZvB6z+vZ0nQ61myP+SRRx555OE/EvR6lgS9zj/XWmuttdbaP4Vp+0Jh2u4/9dNPP/300z/j39So+DflP3/66aeffuo/yMrVAbJy5T9iM3iL2AzePzTRRBNNNNE/2r5QmLYvxD/avlCYti+kP7Qk6PUsCeo/ZOXqAFm56j8gK1cHyMrlPxsV/6ZGxe8/NSr+TY2K7z8Upu0LhWnrP84555xzzuk/BhZYYIEFpj9qVPybGhXvP99777333us/vvfee++95z+HG2644YbrP4cbbrjhhts/aEnQ61kS5D//pkbFv6nhP1cHyMrVAeI/BhZYYIEFxj922WWXXXbpP6iffvrpp+8/8W9qVPyb6j9yHMdxHMfhP7xFbAZvEcs/3cl8SHcyzz/Nh3Qn8yHdP0UTTTTRRLM/wKlR8W9q5D9fKEzbFwrTP42Kf1Oj4u8/gl7PkqDX4z+PPPLII4/sPx555JFHHuk/77333nvv7T/avlCYti/UP6dGxb+pUeE/hxtuuOGGyz+on3766affP0uCXs+SoNc/8W9qVPybuj+fJUGvZ0nQPwKycnWArOw/cGpU/Jsa5T96PUuCXs/iP7JydYCsXI0/2r5QmLYv5D8GFlhggQXmP7xFbAZvEes/fUh3Mh/SzT/XWmuttdbqP/30008//eQ/eIvYDN4i5j/tsssuu+ziP/tCYdq+UOg/cGpU/JsaxT/DtH2hMG3fPzfccMMNN+w/wKlR8W9q1D8oTNsXCtPmP4QQQgghhLA/Zpdddtll5z9IHnnkkUfeP1oS9HqWBO0/oH766aef7j+Nin9To+LfP/qQ7mQ+pOs/6wBZuTpA1j8tCXo9S4LePzIf0p3Mh+Q/snJ1gKxcvT8tCXo9S4LuP5dddtlll+0/T+ZDupP50D8xbV8oTNunPz/99NNPP+0/hBBCCCGE0D+5OkBWrg7AP/FvalT8m9o/snJ1gKxcrT8CsnJ1gKzcP1zEZvAWsak/r2dJ0OtZkj9ml1122WXXPxZYYIEFFug/XMRm8Bax2T+8RWwGbxHbP6EwbV8oTNs/nXPOOeec4z9P5kO6k/nAPzFtXyhM27c/TTTRRBNN5D8xbV8oTNvnP93JfEh3Mu8/RRNNNNFE4z9S8W9qVPzLP1zEZvAWsek/0OtZEvR61j8ssMACCyzgP5dddtlll90/55xzzjnn3D/1008//fTjP58lQa9nSeA/zYd0J/Mh7T9FE0000UTDP3BqVPybGtU/BhZYYIEFlj+llFJKKaXUP1VVVVVVVeU/55xzzjnn7D/dyXxIdzKvP9q+UJi2L7Q/lVJKKaWU4j8avEVsBm/hP3BqVPybGrU//6ZGxb+p0T+sA2Tl6gDpP5vBW8Rm8OY/T+ZDupP54D+ycnWArFzNP6WUUkoppeQ/r2dJ0OtZ0j8ON9xwww3nP1Oj4t/UqOg/FlhggQUW2D/rAFm5OkDmP93JfEh3Mr8/0p3Mh3Qn4z8Q6U7mQ7rDP5Og17Mk6OU/GArT9oXC5D+bwVvEZvDWPxzHcRzHcew/2QzeIjaD5z/7QmHavlDYP0geeeSRR84/uIjN4C1i4z/7QmHavlDIP4wxxhhjjOE/gKxcHSAr1z8GFlhggQW2PwAAAAAAAPA/8yHdyXxI5z/KfEh3Mh/iP69nSdDrWeI/4S1iM3iL6D8HyMrVAbLiPzFtXyhM28c/m8FbxGbwtj96PUuCXs/SPxq8RWwGb8E/3cl8SHcy3z/DtH2hMG3vP2wGbxGbwds/ttZaa6211j+hMG1fKEzrP5kP6U7mQ+o/snJ1gKxcbT+VUkoppZTSP8JbxGbwFuE/m8FbxGbwxj9edtlll13mP1Lxb2pU/Os/XMRm8BaxyT8gK1cHyMrVP4QQQgghhOA/ke5kPqQ76T8avEVsBm/RP2r7QmHavtA/UvFvalT82z+EEEIIIYSgP31IdzIf0u0/VVVVVVVV1T8EZOXqAFnpPwYWWGCBBdY/4S1iM3iL2D/3hcK0faHgP0GvZ0nQ6+k/EZvBW8Rm4D96PUuCXs/CP18oTNsXCuM/XMRm8BaxmT/kkUceeeTRP0UTTTTRRNM/avtCYdq+4D+6k/mQ7mTuP0GvZ0nQ69k/BhZYYIEFhj/GGGOMMcbYP2IzeIvYDO4/hBBCCCGEwD9QP/3000/vP0nQ61kS9Oo/xhhjjDHG6D+HG2644YarP0geeeSRR74/Q2HavlCY5j8S9HqWBL3uP+SRRx555ME/Jej1LAl67T8vu+yyyy7rP4nN4C1iM+g/i39To+Lf1D99SHcyH9LdP6wDZOXqANk/0OtZEvR6xj+5OkBWrg7QP4cbbrjhhrs/o+Lf1Kj45z/VqPg3NSruP9xwww033OA/ynxIdzIf0j/EZvAWsRnsP6WUUkoppcQ/It3JfEh34j8xbV8oTNvXP5HuZD6kO8k/Zpdddtllxz+ycnWArFydP3Un8yHdyew/qJ9++umnzz/pTuZDupPpPwyFaftCYeo/J5poookm6j/GGGOMMca4P0uCXs+SoOc/ke5kPqQ72T8S9HqWBL3ePxzHcRzHcdw/HMdxHMdxzD+ArFwdICvnPxDpTuZDutM/O0BWrg6Q1T8nmmiiiSbKP7k6QFauDuA/snJ1gKxcfT8q/k2Nin/jP4t/U6Pi3+Q/ttZaa6215j9uuOGGG27oPzTRRBNNNOE/55xzzjnnzD8MhWn7QmHaP0geeeSRR+4/\",\"dtype\":\"float64\",\"shape\":[279]},\"__dummy_cat\":[\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \"],\"__label\":[\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \"],\"index\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278]},\"selected\":{\"id\":\"1228\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1227\",\"type\":\"UnionRenderers\"}},\"id\":\"1180\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1172\",\"type\":\"HelpTool\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1071\",\"type\":\"PanTool\"},{\"id\":\"1072\",\"type\":\"WheelZoomTool\"},{\"id\":\"1073\",\"type\":\"BoxZoomTool\"},{\"id\":\"1074\",\"type\":\"SaveTool\"},{\"id\":\"1075\",\"type\":\"ResetTool\"},{\"id\":\"1076\",\"type\":\"HelpTool\"}]},\"id\":\"1077\",\"type\":\"Toolbar\"},{\"attributes\":{\"callback\":null},\"id\":\"1151\",\"type\":\"DataRange1d\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1167\",\"type\":\"PanTool\"},{\"id\":\"1168\",\"type\":\"WheelZoomTool\"},{\"id\":\"1169\",\"type\":\"BoxZoomTool\"},{\"id\":\"1170\",\"type\":\"SaveTool\"},{\"id\":\"1171\",\"type\":\"ResetTool\"},{\"id\":\"1172\",\"type\":\"HelpTool\"}]},\"id\":\"1173\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"1153\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1071\",\"type\":\"PanTool\"},{\"attributes\":{},\"id\":\"1072\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"data_source\":{\"id\":\"1180\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1182\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1183\",\"type\":\"Circle\"},\"selection_glyph\":null,\"view\":{\"id\":\"1185\",\"type\":\"CDSView\"}},\"id\":\"1184\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"lightgrey\"},\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":{\"value\":1.0},\"line_color\":{\"value\":\"black\"},\"line_dash\":[4,4],\"line_width\":{\"value\":2},\"render_mode\":\"css\",\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"1047\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"overlay\":{\"id\":\"1095\",\"type\":\"BoxAnnotation\"}},\"id\":\"1073\",\"type\":\"BoxZoomTool\"},{\"attributes\":{},\"id\":\"1074\",\"type\":\"SaveTool\"},{\"attributes\":{},\"id\":\"1024\",\"type\":\"WheelZoomTool\"},{\"attributes\":{},\"id\":\"1075\",\"type\":\"ResetTool\"},{\"attributes\":{\"callback\":null},\"id\":\"1053\",\"type\":\"DataRange1d\"},{\"attributes\":{},\"id\":\"1076\",\"type\":\"HelpTool\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1067\",\"type\":\"BasicTicker\"}},\"id\":\"1070\",\"type\":\"Grid\"},{\"attributes\":{\"label\":{\"value\":\" \"},\"renderers\":[{\"id\":\"1136\",\"type\":\"GlyphRenderer\"}]},\"id\":\"1145\",\"type\":\"LegendItem\"},{\"attributes\":{\"below\":[{\"id\":\"1061\",\"type\":\"LinearAxis\"}],\"center\":[{\"id\":\"1065\",\"type\":\"Grid\"},{\"id\":\"1070\",\"type\":\"Grid\"},{\"id\":\"1096\",\"type\":\"Legend\"}],\"frame_height\":150,\"frame_width\":200,\"left\":[{\"id\":\"1066\",\"type\":\"LinearAxis\"}],\"plot_height\":300,\"plot_width\":400,\"renderers\":[{\"id\":\"1088\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1051\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1077\",\"type\":\"Toolbar\"},\"toolbar_location\":null,\"x_range\":{\"id\":\"1053\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"1057\",\"type\":\"LinearScale\"},\"y_range\":{\"id\":\"1055\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"1059\",\"type\":\"LinearScale\"}},\"id\":\"1050\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"ticker\":{\"id\":\"1062\",\"type\":\"BasicTicker\"}},\"id\":\"1065\",\"type\":\"Grid\"},{\"attributes\":{\"callback\":null},\"id\":\"1101\",\"type\":\"DataRange1d\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"lightgrey\"},\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":{\"value\":1.0},\"line_color\":{\"value\":\"black\"},\"line_dash\":[4,4],\"line_width\":{\"value\":2},\"render_mode\":\"css\",\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"1191\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"overlay\":{\"id\":\"1047\",\"type\":\"BoxAnnotation\"}},\"id\":\"1025\",\"type\":\"BoxZoomTool\"},{\"attributes\":{},\"id\":\"1019\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"1209\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"items\":[{\"id\":\"1097\",\"type\":\"LegendItem\"}],\"visible\":false},\"id\":\"1096\",\"type\":\"Legend\"},{\"attributes\":{\"children\":[[{\"id\":\"1002\",\"subtype\":\"Figure\",\"type\":\"Plot\"},0,0],[{\"id\":\"1050\",\"subtype\":\"Figure\",\"type\":\"Plot\"},0,1],[{\"id\":\"1098\",\"subtype\":\"Figure\",\"type\":\"Plot\"},1,0],[{\"id\":\"1146\",\"subtype\":\"Figure\",\"type\":\"Plot\"},1,1]]},\"id\":\"1230\",\"type\":\"GridBox\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"x\":{\"field\":\"Rex1\"},\"y\":{\"field\":\"__ECDF\"}},\"id\":\"1183\",\"type\":\"Circle\"},{\"attributes\":{\"fill_color\":{\"value\":\"#1f77b3\"},\"line_color\":{\"value\":\"#1f77b3\"},\"x\":{\"field\":\"Rex1\"},\"y\":{\"field\":\"__ECDF\"}},\"id\":\"1182\",\"type\":\"Circle\"},{\"attributes\":{},\"id\":\"1201\",\"type\":\"Selection\"},{\"attributes\":{\"source\":{\"id\":\"1180\",\"type\":\"ColumnDataSource\"}},\"id\":\"1185\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1093\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1091\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1228\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1105\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1189\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1187\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"ticker\":{\"id\":\"1014\",\"type\":\"BasicTicker\"}},\"id\":\"1017\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1059\",\"type\":\"LinearScale\"},{\"attributes\":{\"items\":[{\"id\":\"1193\",\"type\":\"LegendItem\"}],\"visible\":false},\"id\":\"1192\",\"type\":\"Legend\"},{\"attributes\":{\"axis_label\":\"ECDF\",\"formatter\":{\"id\":\"1091\",\"type\":\"BasicTickFormatter\"},\"ticker\":{\"id\":\"1067\",\"type\":\"BasicTicker\"}},\"id\":\"1066\",\"type\":\"LinearAxis\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"lightgrey\"},\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":{\"value\":1.0},\"line_color\":{\"value\":\"black\"},\"line_dash\":[4,4],\"line_width\":{\"value\":2},\"render_mode\":\"css\",\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"1095\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"x\":{\"field\":\"Prdm14\"},\"y\":{\"field\":\"__ECDF\"}},\"id\":\"1087\",\"type\":\"Circle\"},{\"attributes\":{},\"id\":\"1026\",\"type\":\"SaveTool\"},{\"attributes\":{},\"id\":\"1062\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"1057\",\"type\":\"LinearScale\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1023\",\"type\":\"PanTool\"},{\"id\":\"1024\",\"type\":\"WheelZoomTool\"},{\"id\":\"1025\",\"type\":\"BoxZoomTool\"},{\"id\":\"1026\",\"type\":\"SaveTool\"},{\"id\":\"1027\",\"type\":\"ResetTool\"},{\"id\":\"1028\",\"type\":\"HelpTool\"}]},\"id\":\"1029\",\"type\":\"Toolbar\"},{\"attributes\":{\"label\":{\"value\":\" \"},\"renderers\":[{\"id\":\"1184\",\"type\":\"GlyphRenderer\"}]},\"id\":\"1193\",\"type\":\"LegendItem\"},{\"attributes\":{},\"id\":\"1227\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"below\":[{\"id\":\"1157\",\"type\":\"LinearAxis\"}],\"center\":[{\"id\":\"1161\",\"type\":\"Grid\"},{\"id\":\"1166\",\"type\":\"Grid\"},{\"id\":\"1192\",\"type\":\"Legend\"}],\"frame_height\":150,\"frame_width\":200,\"left\":[{\"id\":\"1162\",\"type\":\"LinearAxis\"}],\"plot_height\":300,\"plot_width\":400,\"renderers\":[{\"id\":\"1184\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1147\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1173\",\"type\":\"Toolbar\"},\"toolbar_location\":null,\"x_range\":{\"id\":\"1149\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"1153\",\"type\":\"LinearScale\"},\"y_range\":{\"id\":\"1151\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"1155\",\"type\":\"LinearScale\"}},\"id\":\"1146\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"label\":{\"value\":\" \"},\"renderers\":[{\"id\":\"1088\",\"type\":\"GlyphRenderer\"}]},\"id\":\"1097\",\"type\":\"LegendItem\"},{\"attributes\":{\"axis_label\":\"mRNA count\",\"formatter\":{\"id\":\"1189\",\"type\":\"BasicTickFormatter\"},\"ticker\":{\"id\":\"1158\",\"type\":\"BasicTicker\"}},\"id\":\"1157\",\"type\":\"LinearAxis\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1019\",\"type\":\"BasicTicker\"}},\"id\":\"1022\",\"type\":\"Grid\"},{\"attributes\":{\"callback\":null},\"id\":\"1055\",\"type\":\"DataRange1d\"},{\"attributes\":{\"callback\":null},\"id\":\"1103\",\"type\":\"DataRange1d\"},{\"attributes\":{},\"id\":\"1107\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1155\",\"type\":\"LinearScale\"},{\"attributes\":{\"below\":[{\"id\":\"1013\",\"type\":\"LinearAxis\"}],\"center\":[{\"id\":\"1017\",\"type\":\"Grid\"},{\"id\":\"1022\",\"type\":\"Grid\"},{\"id\":\"1048\",\"type\":\"Legend\"}],\"frame_height\":150,\"frame_width\":200,\"left\":[{\"id\":\"1018\",\"type\":\"LinearAxis\"}],\"plot_height\":300,\"plot_width\":400,\"renderers\":[{\"id\":\"1040\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1003\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1029\",\"type\":\"Toolbar\"},\"toolbar_location\":null,\"x_range\":{\"id\":\"1005\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"1009\",\"type\":\"LinearScale\"},\"y_range\":{\"id\":\"1007\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"1011\",\"type\":\"LinearScale\"}},\"id\":\"1002\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"text\":\"Prdm14\"},\"id\":\"1051\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"1028\",\"type\":\"HelpTool\"},{\"attributes\":{\"axis_label\":\"mRNA count\",\"formatter\":{\"id\":\"1093\",\"type\":\"BasicTickFormatter\"},\"ticker\":{\"id\":\"1062\",\"type\":\"BasicTicker\"}},\"id\":\"1061\",\"type\":\"LinearAxis\"},{\"attributes\":{\"below\":[{\"id\":\"1109\",\"type\":\"LinearAxis\"}],\"center\":[{\"id\":\"1113\",\"type\":\"Grid\"},{\"id\":\"1118\",\"type\":\"Grid\"},{\"id\":\"1144\",\"type\":\"Legend\"}],\"frame_height\":150,\"frame_width\":200,\"left\":[{\"id\":\"1114\",\"type\":\"LinearAxis\"}],\"plot_height\":300,\"plot_width\":400,\"renderers\":[{\"id\":\"1136\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1099\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1125\",\"type\":\"Toolbar\"},\"toolbar_location\":null,\"x_range\":{\"id\":\"1101\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"1105\",\"type\":\"LinearScale\"},\"y_range\":{\"id\":\"1103\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"1107\",\"type\":\"LinearScale\"}},\"id\":\"1098\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"1027\",\"type\":\"ResetTool\"},{\"attributes\":{},\"id\":\"1200\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"callback\":null,\"data\":{\"Nanog\":[39,33,68,88,41,103,27,15,153,127,73,5,64,104,202,216,51,293,4,1,5,143,77,27,156,0,100,101,33,120,101,57,97,121,228,79,48,0,34,107,110,408,231,153,138,10,137,69,174,61,32,84,67,141,24,94,246,80,69,11,20,10,13,44,68,251,127,74,156,44,61,33,39,175,128,24,67,274,78,49,4,143,193,82,10,176,86,107,130,149,31,40,177,37,145,1,86,113,115,139,68,83,64,94,117,26,108,120,84,2,144,62,55,12,21,39,33,14,103,78,82,46,47,79,1,6,24,116,104,63,59,132,125,153,85,104,52,140,128,22,35,2,162,236,140,14,3,132,120,30,84,200,74,154,81,77,59,88,80,10,175,19,124,60,87,194,29,119,67,8,38,94,28,22,66,6,90,77,92,118,171,29,32,11,77,37,198,134,14,227,190,40,1,50,65,19,64,154,65,25,71,129,114,123,156,8,132,136,80,162,53,87,72,64,19,134,10,65,109,96,105,21,20,116,239,0,83,263,67,0,29,66,191,30,196,49,186,181,104,127,66,11,19,155,252,160,38,41,6,195,55,24,94,11,66,73,263,168,128,18,58,72,105,118,23,212,76,32,29,104,2,76,148,187,141,3,8,74,104],\"__ECDF\":{\"__ndarray__\":\"r2dJ0OtZ0j+5OkBWrg7QPxzHcRzHcdw/B8jK1QGy4j8q/k2Nin/TP9q+UJi2L+Q/J5poookmyj8avEVsBm/BP9daa6211uo/vvfee++95z9IHnnkkUfeP7JydYCsXK0/rANk5eoA2T/AqVHxb2rkP0geeeSRR+4/LQl6PUuC7j8gK1cHyMrVP42Kf1Oj4u8/XMRm8BaxqT+vZ0nQ61mSP93JfEh3Mq8/J5poookm6j+Nin9To+LfP/FvalT8m8o/hxtuuOGG6z+ycnWArFxtP4Jez5Kg1+M/9dNPP/304z+fJUGvZ0nQP5vBW8Rm8OY/aEnQ61kS5D+bwVvEZvDWPxDpTuZDuuM/8yHdyXxI5z8S9HqWBL3uP/eFwrR9oeA/i39To+Lf1D+ycnWArFx9P0/mQ7qT+dA/VVVVVVVV5T8gK1cHyMrlPwAAAAAAAPA/hWn7QmHa7j9J0OtZEvTqP3bZZZddduk/BhZYYIEFtj8EZOXqAFnpP82HdCfzId0/HMdxHMdx7D8WWGCBBRbYPxL0epYEvc4/chzHcRzH4T+HG2644YbbP0GvZ0nQ6+k/Zpdddtllxz/SncyHdCfjP93JfEh3Mu8/3HDDDTfc4D+ycnWArFzdP/FvalT8m7o/2r5QmLYvxD+bwVvEZvC2P93JfEh3Mr8/9dNPP/300z8CsnJ1gKzcP1A//fTTT+8/MW1fKEzb5z8S9HqWBL3eP/qQ7mQ+pOs/2r5QmLYv1D/7QmHavlDYP4QQQgghhNA/lVJKKaWU0j+PPPLII4/sPxZYYIEFFug/MW1fKEzbxz9sBm8Rm8HbPxsV/6ZGxe8/EZvBW8Rm4D9walT8mxrVP4cbbrjhhqs/mQ/pTuZD6j+XXXbZZZftP6dGxb+pUeE/MW1fKEzbtz91J/Mh3cnsPz3yyCOPPOI/yMrVAbJy5T/hLWIzeIvoP2Tl6gBZueo/SB555JFHzj9fKEzbFwrTP+ecc8455+w/GrxFbAZv0T9/+umnn37qPwYWWGCBBZY/r2dJ0OtZ4j+ToNezJOjlP3iL2AzeIuY/6U7mQ7qT6T/nnHPOOefcP4wxxhhjjOE/ke5kPqQ72T9FE0000UTjP9DrWRL0euY/XMRm8BaxyT87QFauDpDlPw433HDDDec/5JFHHnnk4T+EEEIIIYSgPwyFaftCYeo/4S1iM3iL2D/Q61kS9HrWP0geeeSRR74/cGpU/JsaxT96PUuCXs/SP2r7QmHavtA/uTpAVq4OwD9NNNFEE03kP4QQQgghhOA/GrxFbAZv4T/AqVHxb2rUP6WUUkoppdQ/avtCYdq+4D9cxGbwFrGZP4QQQgghhLA/+0Jh2r5QyD/rAFm5OkDmPzIf0p3Mh+Q/xhhjjDHG2D9ml1122WXXP1Oj4t/UqOg/S4Jez5Kg5z+8RWwGbxHrP8p8SHcyH+I/pZRSSiml5D8GFlhggQXWP1zEZvAWsek/ic3gLWIz6D8GFlhggQXGPzTRRBNNNNE/r2dJ0OtZoj9S8W9qVPzrP/jee++99+4/zjnnnHPO6T+EEEIIIYTAPwYWWGCBBaY/xhhjjDHG6D+ArFwdICvnP7JydYCsXM0/VwfIytUB4j/VqPg3NSruP/jee++9994/L7vssssu6z800UQTTTThP7k6QFauDuA/S4Jez5Kg1z96PUuCXs/iP0/mQ7qT+eA/xhhjjDHGuD8CsnJ1gKzsP69nSdDrWcI/2QzeIjaD5z8xbV8oTNvXPyLdyXxId+I/CtP2hcK07T+HG2644YbLPyhM2xcK0+Y/UvFvalT82z9FE0000USzP+SRRx555NE/uIjN4C1i4z+8RWwGbxHLP9DrWRL0esY/8W9qVPyb2j8avEVsBm+xP+2yyy677OI/LLDAAgss4D9fKEzbFwrjP0Nh2r5QmOY/qlHxb2pU7D9S8W9qVPzLP93JfEh3Ms8/hxtuuOGGuz+fJUGvZ0ngP/+mRsW/qdE/YjN4i9gM7j+sA2Tl6gDpP0/mQ7qT+cA/oH766aef7j+ycnWArFztP0UTTTTRRNM/snJ1gKxcnT87QFauDpDVP0GvZ0nQ69k/ej1Lgl7Pwj922WWXXXbZP6EwbV8oTOs/J5poookm2j+R7mQ+pDvJP5dddtlll90/brjhhhtu6D8GFlhggQXmP2aXXXbZZec/bAZvEZvB6z/avlCYti+0PzmO4ziO4+g/ke5kPqQ76T/CW8Rm8BbhP8Rm8BaxGew/6wBZuTpA1j+VUkoppZTiP31IdzIf0t0/XMRm8Bax2T9FE0000UTDPx555JFHHuk/XMRm8BaxuT8MhWn7QmHaP6211lprreU/nXPOOeec4z9walT8mxrlPztAVq4OkMU/pZRSSimlxD9edtlll13mP2pU/JsaFe8/BhZYYIEFhj//pkbFv6nhPzUq/k2Niu8/N9xwww033D+ycnWArFyNPxzHcRzHccw/11prrbXW2j8l6PUsCXrtP31IdzIf0s0/77333nvv7T9VVVVVVVXVP82HdCfzIe0/WhL0epYE7T8YCtP2hcLkP6Pi39So+Oc/vEVsBm8R2z8cx3Ecx3G8PxDpTuZDusM/FKbtC4Vp6z/DtH2hMG3vP99777333us/ynxIdzIf0j8Q6U7mQ7rTP69nSdDrWbI/fUh3Mh/S7T+21lprrbXWP8YYY4wxxsg/Kv5NjYp/4z+ycnWArFy9P6EwbV8oTNs/LQl6PUuC3j+on3766afvPzfccMMNN+w/+0Jh2r5Q6D/kkUceeeTBP4CsXB0gK9c/YjN4i9gM3j/j39So+DflP7bWWmutteY/m8FbxGbwxj+6k/mQ7mTuP8O0faEwbd8/qJ9++umnzz/nnHPOOefMP4t/U6Pi3+Q/2r5QmLYvpD+on3766affP/FvalT8m+o/P/30008/7T+0JOj1LAnqPzFtXyhM26c/cGpU/JsatT/dyXxIdzLfP/30008//eQ/\",\"dtype\":\"float64\",\"shape\":[279]},\"__dummy_cat\":[\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \"],\"__label\":[\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \",\" \"],\"index\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278]},\"selected\":{\"id\":\"1201\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1200\",\"type\":\"UnionRenderers\"}},\"id\":\"1036\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1067\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"1115\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"1110\",\"type\":\"BasicTicker\"},{\"attributes\":{\"text\":\"Rest\"},\"id\":\"1099\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"1158\",\"type\":\"BasicTicker\"},{\"attributes\":{\"data_source\":{\"id\":\"1084\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1086\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1087\",\"type\":\"Circle\"},\"selection_glyph\":null,\"view\":{\"id\":\"1089\",\"type\":\"CDSView\"}},\"id\":\"1088\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"axis_label\":\"ECDF\",\"formatter\":{\"id\":\"1043\",\"type\":\"BasicTickFormatter\"},\"ticker\":{\"id\":\"1019\",\"type\":\"BasicTicker\"}},\"id\":\"1018\",\"type\":\"LinearAxis\"},{\"attributes\":{\"axis_label\":\"mRNA count\",\"formatter\":{\"id\":\"1141\",\"type\":\"BasicTickFormatter\"},\"ticker\":{\"id\":\"1110\",\"type\":\"BasicTicker\"}},\"id\":\"1109\",\"type\":\"LinearAxis\"},{\"attributes\":{\"axis_label\":\"ECDF\",\"formatter\":{\"id\":\"1139\",\"type\":\"BasicTickFormatter\"},\"ticker\":{\"id\":\"1115\",\"type\":\"BasicTicker\"}},\"id\":\"1114\",\"type\":\"LinearAxis\"}],\"root_ids\":[\"1233\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n", " var render_items = [{\"docid\":\"5826a1ec-19cf-4e0f-8a1a-0ac57003a5ee\",\"roots\":{\"1233\":\"371ce6cb-beca-43ea-ad88-cb4d96103f11\"}}];\n", " root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n", "\n", " }\n", " if (root.Bokeh !== undefined) {\n", " embed_document(root);\n", " } else {\n", " var attempts = 0;\n", " var timer = setInterval(function(root) {\n", " if (root.Bokeh !== undefined) {\n", " clearInterval(timer);\n", " embed_document(root);\n", " } else {\n", " attempts++;\n", " if (attempts > 100) {\n", " clearInterval(timer);\n", " console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n", " }\n", " }\n", " }, 10, root)\n", " }\n", "})(window);" ], "application/vnd.bokehjs_exec.v0+json": "" }, "metadata": { "application/vnd.bokehjs_exec.v0+json": { "id": "1233" } }, "output_type": "display_data" } ], "source": [ "genes = [\"Nanog\", \"Prdm14\", \"Rest\", \"Rex1\"]\n", "\n", "plots = [\n", " bokeh_catplot.ecdf(\n", " data=df[gene],\n", " val=gene,\n", " x_axis_label=\"mRNA count\",\n", " title=gene,\n", " frame_height=150,\n", " frame_width=200,\n", " )\n", " for gene in genes\n", "]\n", "\n", "bokeh.io.show(bokeh.layouts.gridplot(plots, ncols=2))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Rex1 clearly is bimodal, and Nanog appears to have a second where the ECDF reaches a value of about 0.4, which is what we see in the MLE estimates in the mixture model. Rest and Prdm14 both appear to be unimodal, agreeing with what we saw with the AIC analysis.\n", "\n", "Note that this underscores something we've been stressing all term. You should do good exploratory data analysis first, and the EDA often tells much of the story!" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Caveat\n", "\n", "Remember, though, that we did *not* take into account that the measurements of the four genes were done in the same cells. We modeled that when we presented the mixture models at the beginning of this lesson. The analysis of a more complicated model with MLE proved to be out of reach due to computational difficulty. So, we should not make strong conclusions about what the relative quality of the mixture of single Negative Binomial models mean in this context. We will address these kinds of modeling issues next term." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## AIC for the spindle model\n", "\n", "We can do a similar analysis for the two competing models for mitotic spindle size. We need our functions from earlier." ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "def theor_spindle_length(gamma, phi, d):\n", " \"\"\"Compute spindle length using mathematical model\"\"\"\n", " return gamma * d / np.cbrt(1 + (gamma * d / phi)**3)\n", "\n", "\n", "def log_likelihood(params, d, ell):\n", " \"\"\"Log likelihood of spindle length model.\"\"\"\n", " gamma, phi, sigma = params\n", "\n", " if gamma <= 0 or gamma > 1 or phi <= 0:\n", " return -np.inf\n", "\n", " mu = theor_spindle_length(gamma, phi, d)\n", " return np.sum(st.norm.logpdf(ell, mu, sigma))\n", "\n", "\n", "def spindle_mle(d, ell):\n", " \"\"\"Compute MLE for parameters in spindle length model.\"\"\"\n", " with warnings.catch_warnings():\n", " warnings.simplefilter(\"ignore\")\n", "\n", " res = scipy.optimize.minimize(\n", " fun=lambda params, d, ell: -log_likelihood(params, d, ell),\n", " x0=np.array([0.5, 35, 5]),\n", " args=(d, ell),\n", " method='Powell'\n", " )\n", "\n", " if res.success:\n", " return res.x\n", " else:\n", " raise RuntimeError('Convergence failed with message', res.message) " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We can now perform MLE to get the parameters for each model and store them in a Pandas Series for convenience." ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "df = pd.read_csv(\"../data/good_invitro_droplet_data.csv\", comment=\"#\")\n", "\n", "mle_1 = df[\"Spindle Length (um)\"].apply([np.mean, np.std]).values\n", "mle_2 = spindle_mle(\n", " df[\"Droplet Diameter (um)\"].values, df[\"Spindle Length (um)\"].values\n", ")\n", "\n", "s = pd.Series(\n", " index=[\"phi_1\", \"sigma_1\", \"gamma\", \"phi_2\", \"sigma_2\"],\n", " data=np.concatenate((mle_1, mle_2)),\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Next, we can compute the log likelihood evaluated at the MLE." ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "phi_1 32.864030\n", "sigma_1 4.788240\n", "gamma 0.860475\n", "phi_2 38.231250\n", "sigma_2 3.753422\n", "log_like_1 -1999.517925\n", "log_like_2 -1837.158982\n", "dtype: float64" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "s[\"log_like_1\"] = st.norm.logpdf(df[\"Spindle Length (um)\"], s[\"phi_1\"], s[\"sigma_1\"]).sum()\n", "s[\"log_like_2\"] = log_likelihood(\n", " s[[\"gamma\", \"phi_2\", \"sigma_2\"]],\n", " df[\"Droplet Diameter (um)\"],\n", " df[\"Spindle Length (um)\"],\n", ")\n", "\n", "s" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The log likeihood for model 2, with spindle size depending on droplet diameter, is much greater than for model 1. And now we can compute the AIC,noting that there are two parameters for model 1 and three for model 2." ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "phi_1 32.864030\n", "sigma_1 4.788240\n", "gamma 0.860475\n", "phi_2 38.231250\n", "sigma_2 3.753422\n", "log_like_1 -1999.517925\n", "log_like_2 -1837.158982\n", "AIC_1 4003.035850\n", "AIC_2 3680.317964\n", "dtype: float64" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "s[\"AIC_1\"] = -2 * (s['log_like_1'] - 2)\n", "s[\"AIC_2\"] = -2 * (s['log_like_2'] - 3)\n", "\n", "s" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "There is a massive disparity in the AICs, so we know that model 2 is strongly preferred. Nonetheless, we can compute the Akaike weight for model 1 to compare." ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "phi_1 3.286403e+01\n", "sigma_1 4.788240e+00\n", "gamma 8.604745e-01\n", "phi_2 3.823125e+01\n", "sigma_2 3.753422e+00\n", "log_like_1 -1.999518e+03\n", "log_like_2 -1.837159e+03\n", "AIC_1 4.003036e+03\n", "AIC_2 3.680318e+03\n", "w_single 8.369539e-71\n", "dtype: float64" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "AIC_max = max(s[['AIC_1', 'AIC_2']])\n", "numerator = np.exp(-(s.loc['AIC_1'] - AIC_max)/2)\n", "denominator = numerator + np.exp(-(s['AIC_2'] - AIC_max)/2)\n", "s['w_single'] = numerator / denominator\n", "\n", "s" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Model 1 is completely out of the question, with a tiny Akaike weight!" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Computing environment" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPython 3.7.5\n", "IPython 7.9.0\n", "\n", "numpy 1.17.4\n", "pandas 0.24.2\n", "scipy 1.3.1\n", "bokeh 1.4.0\n", "bokeh_catplot 0.1.6\n", "bebi103 0.0.45\n", "jupyterlab 1.2.3\n" ] } ], "source": [ "%load_ext watermark\n", "%watermark -v -p numpy,pandas,scipy,bokeh,bokeh_catplot,bebi103,jupyterlab" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.5" } }, "nbformat": 4, "nbformat_minor": 4 }