History of fly_optic_lobe
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] New entry: fly_optic_lobe cc963b5
diff --git a/entries/fly_optic_lobe/__init__.py b/entries/fly_optic_lobe/__init__.py
new file mode 100644
index 0000000..3212875
--- /dev/null
+++ b/entries/fly_optic_lobe/__init__.py
@@ -0,0 +1,168 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from .. import *
+import re
+import numpy as np
+import pandas as pd
+import pyarrow.feather
+import pyarrow.ipc
+
+title = "Male fly optic lobe (2025)"
+description = r"""The connectome of the right optic lobe (the visual system) of an adult male fruit fly *Drosophila melanogaster*, together with the parts of the central brain that it connects to, reconstructed by the FlyEM project at the Janelia Research Campus and collaborators from focused ion beam scanning electron microscopy (FIB-SEM) images, with automated segmentation and synapse detection followed by extensive manual proofreading. The authors identified over 700 cell types among the roughly 53,000 neurons of the optic lobe. Nodes are neurons (segments of the reconstruction), and a directed edge $(i, j)$ indicates chemical synapses from neuron $i$ (presynaptic) to neuron $j$ (postsynaptic). The data come from the two releases of the data set that the authors exported as flat files, v1.1 (September 2024, with further proofreading and refined annotations) and v1.0.1 (August 2024).
+
+Only synapses detected with a confidence of at least 0.5 are included, as in the authors' published data. There are six networks, named `<release>_<nodes>_<threshold>`:
+
+- `*_primary_min5`, `*_primary_all`: the networks between the proofread bodies, from the authors' "primary-only" connection table. Its nodes are the bodies (53,735 in v1.1, 53,420 in v1.0.1) with the proofreading status "Roughly traced", "Prelim Roughly traced", "Leaves", "PRT Orphan" (together the "Traced" neurons of neuPrint) or "Primary Anchor" (large segments that were not fully traced); the upstream documentation does not describe this table, this is what it contains. In `*_min5` only the pairs of neurons with at least five synapses are considered connected, since weak connections are unreliable, and for consistency with the `flywire`, `fly_banc` and `fly_male_cns` entries; in `*_all` all connections are included. In `v1_1_primary_min5`, these are 1,555,385 of the 6,497,418 connected pairs, carrying 17.6 million of their 26.6 million synapses.
+- `*_segments_all`: all 10.3 million segments of the reconstruction, most of them small fragments that could not be attached to a neuron, with all their connections (22.3 million connected pairs and 49.5 million synapses). The table also contains a few connections with the unsegmented background (body 0; 198 pairs with 207 synapses), which are left out. To keep these networks manageable, their nodes only have the properties `body_id`, `status_label`, `type`, `instance`, `class`, `pre` and `post`.
+
+All networks of a release and node set have the same nodes, so that neurons without connections (with at least five synapses, in `*_min5`) are isolated nodes. Self-loops are synapses of a neuron onto itself (autapses). The annotations and neurotransmitter predictions are those of the corresponding release, from its neuPrint tables.
+
+The networks have the following properties:
+
+- `body_id`: the neuron's identifier in the data set (neuPrint body ID).
+- `status_label`: the proofreading status of the body (e.g. `Roughly traced`, `Primary Anchor`; empty for unproofread fragments).
+- `type`: the cell type of the neuron (e.g. `Mi1`, `T4a`); empty if not assigned.
+- `instance`: the neuron's instance name, typically the cell type followed by the side (e.g. `Mi1_R`).
+- `class`: the coarse class of the neuron (e.g. `optic`, `visual_projection`, `visual_centrifugal`, `sensory`); empty if not assigned.
+- `flywire_type`: the matched cell type in the female brain connectome FlyWire (see the `flywire` entry); empty if not assigned (always empty in v1.0.1, which has no such annotations).
+- `hemilineage`: the developmental hemilineage of the neuron; empty if not assigned (always empty in v1.0.1).
+- `neurotransmitter`: the consensus neurotransmitter of the neuron's cell type, from the predictions and the literature (`unclear` if ambiguous); empty if not available.
+- `predicted_nt`, `predicted_nt_confidence`: the neurotransmitter predicted from the neuron's own synapses, and the confidence of that prediction (empty and NaN if not available).
+- `hex1`, `hex2`: the coordinates of the column of the medulla's hexagonal grid (i.e. the point of the visual field) assigned to the neuron, for the columnar cell types (-1 if not assigned).
+- `soma_x`, `soma_y`, `soma_z`: the position of the neuron's soma in the volume, in voxels of 8 nm (NaN if not known).
+- `pre`, `post`: the numbers of presynaptic sites and of postsynaptic densities of the neuron.
+- `weight` (edges): the number of synapses from $i$ to $j$."""
+tags = ['Biological', 'Connectome', 'Weighted', 'Metadata']
+url = 'https://www.janelia.org/project-team/flyem/optic-lobe'
+citation = [('A. Nern et al., "Connectome-driven neural inventory of a complete visual system." Nature 641, 1225-1237 (2025)', 'https://doi.org/10.1038/s41586-025-08746-0')]
+icon_hash = None
+upstream_license = ("CC BY 4.0", "https://creativecommons.org/licenses/by/4.0/")
+upstream_prefix = 'https://storage.googleapis.com/flyem-optic-lobe'
+# release: (directory, file tag)
+_releases = {"v1_1": ("v1.1/optic-lobe-v1.1", "2024-09-11-a7d912"),
+ "v1_0_1": ("v1.0/optic-lobe-v1.0.1", "2023-10-31-27a052")}
+def _files(release, nodes):
+ d, tag = _releases[release]
+ suffix = "-primary-only" if nodes == "primary" else ""
+ return (f"{d}-flat-connectome/connectome-weights-{tag}-minconf-0.5-ol{suffix}.feather",
+ f"{d}-flat-connectome/body-stats-{tag}-minconf-0.5-ol.feather",
+ f"{d}-neuprint-tables/Neuprint_Neurons.feather")
+files = [(_files(release, nodes), f"{release}_{nodes}_{thres}", None)
+ for release in _releases
+ for nodes, thres in [("primary", "min5"), ("primary", "all"),
+ ("segments", "all")]]
+
+# minimum number of synapses between two neurons for them to be connected
+min_synapses = {"min5": 5, "all": 0}
+
+# the proofreading statuses of the bodies of the "primary-only" table
+primary_status = ["Roughly traced", "Prelim Roughly traced", "Leaves",
+ "PRT Orphan", "Primary Anchor"]
+
+def fetch_upstream(force=False):
+ return fetch_upstream_files(__name__.split(".")[-1], upstream_prefix, files,
+ force)
+
+# columns of the neuPrint table: property name
+neuron_props = {"flywireType": "flywire_type", "hemilineage": "hemilineage",
+ "consensusNt": "neurotransmitter", "predictedNt": "predicted_nt"}
+
+def read_neurons(name, fname, ids):
+ "Annotations of the bodies `ids` from the neuPrint table of neurons."
+ with open_upstream_file(name, fname, "rb", seekable=True) as f:
+ reader = pyarrow.ipc.open_file(f)
+ cols = {c.split(":")[0]: c for c in reader.schema.names
+ if not c.startswith(":")}
+ want = ["bodyId"] + list(neuron_props) + \
+ ["predictedNtConfidence", "assignedOlHex1", "assignedOlHex2",
+ "somaLocation"]
+ parts = []
+ for i in range(reader.num_record_batches):
+ b = reader.get_batch(i)
+ b = b.select([cols[c] for c in want if c in cols]).to_pandas()
+ b.columns = [c.split(":")[0] for c in b.columns]
+ # only bodies with some annotation are kept
+ b = b[b.drop(columns=["bodyId"]).notna().any(axis=1)]
+ parts.append(b)
+ nt = pd.concat(parts)
+ nt = nt.set_index("bodyId").reindex(ids)
+ for c in want[1:]:
+ if c not in nt.columns:
+ nt[c] = None
+ return nt
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+ name = __name__.split(".")[-1]
+ for (fweights, fstats, fneurons), alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ with open_upstream_file(name, fstats, "rb", seekable=True) as f:
+ bodies = pd.read_feather(f, columns=["body", "pre", "post", "status",
+ "class", "type", "instance"])
+ if "_primary_" in alt:
+ bodies = bodies[bodies["status"].isin(primary_status)]
+ bodies = bodies.reset_index(drop=True)
+ ids = bodies["body"].to_numpy()
+ # the networks of all segments have only the basic properties
+ full = "_primary_" in alt
+ if full:
+ nt = read_neurons(name, fneurons, ids)
+
+ g = Graph()
+ g.add_vertex(len(bodies))
+ g.vp.body_id = g.new_vp("int64_t", vals=ids.astype("int64"))
+ g.vp.status_label = g.new_vp("string", vals=bodies["status"].astype(str).replace("nan", "").tolist())
+ for c in ["type", "instance", "class"]:
+ g.vp[c] = g.new_vp("string", vals=bodies[c].fillna("").astype(str).tolist())
+ if full:
+ for c, p in neuron_props.items():
+ g.vp[p] = g.new_vp("string", vals=nt[c].fillna("").astype(str).tolist())
+ g.vp.predicted_nt_confidence = \
+ g.new_vp("double", vals=nt["predictedNtConfidence"].astype(float).to_numpy())
+ for i in [1, 2]:
+ h = nt[f"assignedOlHex{i}"].astype(float).fillna(-1)
+ g.vp[f"hex{i}"] = g.new_vp("int", vals=h.astype(int).to_numpy())
+ pos = np.full((len(bodies), 3), np.nan)
+ for i, s in enumerate(nt["somaLocation"].to_numpy()):
+ if isinstance(s, str) and s:
+ m = re.match(r"\{x:(-?[\d.]+), y:(-?[\d.]+), z:(-?[\d.]+)\}", s)
+ pos[i] = [float(x) for x in m.groups()]
+ for j, c in enumerate("xyz"):
+ g.vp[f"soma_{c}"] = g.new_vp("double", vals=pos[:, j])
+ g.vp.pre = g.new_vp("int", vals=bodies["pre"].to_numpy())
+ g.vp.post = g.new_vp("int", vals=bodies["post"].to_numpy())
+ if full:
+ del nt
+
+ with open_upstream_file(name, fweights, "rb", seekable=True) as f:
+ w = pyarrow.feather.read_table(f, columns=["body_pre", "body_post", "weight"]).to_pandas()
+ # body 0 is the unsegmented background
+ w = w[(w["weight"] >= min_synapses[alt.split("_")[-1]]) &
+ (w["body_pre"] != 0) & (w["body_post"] != 0)]
+ idx = pd.Series(np.arange(len(bodies)), index=ids)
+ s = idx.reindex(w["body_pre"].to_numpy()).to_numpy()
+ t = idx.reindex(w["body_post"].to_numpy()).to_numpy()
+ if np.isnan(s).any() or np.isnan(t).any():
+ raise ValueError(f"{alt}: connections of unknown bodies")
+ g.add_edge_list(np.column_stack((s, t)).astype("int64"))
+ g.ep.weight = g.new_ep("int", vals=w["weight"].to_numpy())
+ yield alt, g