History of microns_mm3
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] Add the date when each entry was added, and remove icon_hash beff7c9
The dates are those of the commits that first added the entries' files (2020-08-11 for the entries of the initial commit); renamed entries keep the date of their first name. Changed: none
diff --git a/entries/microns_mm3/__init__.py b/entries/microns_mm3/__init__.py
index be96e17..be823a3 100644
--- a/entries/microns_mm3/__init__.py
+++ b/entries/microns_mm3/__init__.py
@@ -65,7 +65,7 @@ url = ['https://zenodo.org/records/16744240', 'https://zenodo.org/records/138494
'https://www.microns-explorer.org/cortical-mm3']
citation = [('The MICrONS Consortium, "Functional connectomics spanning multiple areas of mouse visual cortex." Nature 640, 435-447 (2025)', 'https://doi.org/10.1038/s41586-025-08790-w'),
('M. W. Reimann, "Connectomics of (part of) the MICrONS mm3 dataset." Zenodo, version 3.0.0 (2025)', 'https://doi.org/10.5281/zenodo.16744240')]
-icon_hash = None
+date_added = '2026-10-27'
upstream_license = ("CC BY 4.0", "https://creativecommons.org/licenses/by/4.0/")
# The three versions of the compilation (Zenodo records), with the
# materialization of the MICrONS data they are based on
[entries] Prefix the properties in the descriptions with (Node), (Edge) or (Graph) bc3dbbd
The lists of properties at the end of the descriptions marked edge properties with "Edge" but left node properties unmarked, or grouped them in "Nodes: `a`, ...; `b`, ..." items, so the kind of a property was often unclear. Every item now starts with (Node), (Edge) or (Graph), followed by the property names, a colon and the explanation, one item per property or group of properties; qualifications (kinds of nodes, networks) follow the names in parentheses. Properties that were only named got a short explanation, and a few items that named the wrong kind of property were corrected.
diff --git a/entries/microns_mm3/__init__.py b/entries/microns_mm3/__init__.py
index e587892..be96e17 100644
--- a/entries/microns_mm3/__init__.py
+++ b/entries/microns_mm3/__init__.py
@@ -42,19 +42,19 @@ All versions leave out a few hundred cell bodies associated with several cell ty
The networks have the following properties:
-- `root_id`: the neuron's identifier (root ID) in the MICrONS data of that version.
-- `cell_type`: the type of the neuron, predicted from features of its soma and nucleus: in `v1412` and `v1181`, the morphological type (m-type; e.g. `L2a`, `L4b`, `L5ET`, `L6tall-a` for excitatory neurons; `PTC`, `DTC`, `STC` and `ITC` for perisomatic-targeting, distal-targeting, sparsely-targeting and inhibitory-targeting inhibitory neurons); in `v117`, the coarse cell class (e.g. `23P`, `4P`, `5P_IT`, `6CT`, `BC`, `MC`).
-- `classification_system` (`v1412` and `v1181`): `excitatory_neuron` or `inhibitory_neuron`.
-- `synapse_class` (`v1412`): whether the neuron is excitatory (`EXC`) or inhibitory (`INH`).
-- `layer` (`v1412`): the cortical layer of the neuron (`2` to `6`), derived from its type; empty for inhibitory neurons.
-- `tentative_region` (`v117`): the visual area of the neuron (`visp`, `visrl`, `visal` or `vislm`), which the author of the compilation warns is only a rough estimate.
-- `status_axon`, `status_dendrite`: whether the neuron's axon and dendrite were proofread (`t`) or not (`f`), for the neurons in the proofreading table of MICrONS; in `v117`, the proofreading status of the public release of the time: `non` (not proofread), `clean` (false merges removed) or `extended` (also false splits corrected); empty for the neurons that are not in that table, which were not proofread.
-- `volume` (`v1412` and `v1181`): the volume of the neuron's nucleus, in µm³.
-- `x`, `y`, `z`: the position of the neuron's soma, in µm ($y$ increases with the cortical depth).
-- Edge `weight`: the number of synapses from $i$ to $j$ (not in the `_full` networks and in `v1412_control`).
-- Edge `total_size`, `mean_size` (`v1181` and `v117`): the sum and the mean of the sizes of these synapses, as given by the synapse table of MICrONS (in voxels).
-- Edge `synapse_id`, `size` (`_full` networks): the identifier of the synapse in the synapse table of MICrONS, and its size (in voxels).
-- Edge `syn_x`, `syn_y`, `syn_z`, `soma_distance` (`v1181_full` and `v117_full`): the position of the synapse, in µm, and its distance from the soma of the postsynaptic neuron, in µm.
+- (Node) `root_id`: the neuron's identifier (root ID) in the MICrONS data of that version.
+- (Node) `cell_type`: the type of the neuron, predicted from features of its soma and nucleus: in `v1412` and `v1181`, the morphological type (m-type; e.g. `L2a`, `L4b`, `L5ET`, `L6tall-a` for excitatory neurons; `PTC`, `DTC`, `STC` and `ITC` for perisomatic-targeting, distal-targeting, sparsely-targeting and inhibitory-targeting inhibitory neurons); in `v117`, the coarse cell class (e.g. `23P`, `4P`, `5P_IT`, `6CT`, `BC`, `MC`).
+- (Node) `classification_system` (`v1412` and `v1181`): `excitatory_neuron` or `inhibitory_neuron`.
+- (Node) `synapse_class` (`v1412`): whether the neuron is excitatory (`EXC`) or inhibitory (`INH`).
+- (Node) `layer` (`v1412`): the cortical layer of the neuron (`2` to `6`), derived from its type; empty for inhibitory neurons.
+- (Node) `tentative_region` (`v117`): the visual area of the neuron (`visp`, `visrl`, `visal` or `vislm`), which the author of the compilation warns is only a rough estimate.
+- (Node) `status_axon`, `status_dendrite`: whether the neuron's axon and dendrite were proofread (`t`) or not (`f`), for the neurons in the proofreading table of MICrONS; in `v117`, the proofreading status of the public release of the time: `non` (not proofread), `clean` (false merges removed) or `extended` (also false splits corrected); empty for the neurons that are not in that table, which were not proofread.
+- (Node) `volume` (`v1412` and `v1181`): the volume of the neuron's nucleus, in µm³.
+- (Node) `x`, `y`, `z`: the position of the neuron's soma, in µm ($y$ increases with the cortical depth).
+- (Edge) `weight`: the number of synapses from $i$ to $j$ (not in the `_full` networks and in `v1412_control`).
+- (Edge) `total_size`, `mean_size` (`v1181` and `v117`): the sum and the mean of the sizes of these synapses, as given by the synapse table of MICrONS (in voxels).
+- (Edge) `synapse_id`, `size` (`_full` networks): the identifier of the synapse in the synapse table of MICrONS, and its size (in voxels).
+- (Edge) `syn_x`, `syn_y`, `syn_z`, `soma_distance` (`v1181_full` and `v117_full`): the position of the synapse, in µm, and its distance from the soma of the postsynaptic neuron, in µm.
In `v1412_control`, the nodes have the properties `root_id`, `cell_type` (the m-type), `synapse_class` (`EXC`), `layer`, `volume` and `x`, `y`, `z`, as above.
[entries] New entry: microns_mm3 db6bd4c
diff --git a/entries/microns_mm3/__init__.py b/entries/microns_mm3/__init__.py
new file mode 100644
index 0000000..e587892
--- /dev/null
+++ b/entries/microns_mm3/__init__.py
@@ -0,0 +1,234 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from .. import *
+import numpy as np
+import pandas as pd
+import h5py
+import io
+import importlib
+import pickle
+import struct
+
+title = "MICrONS mouse visual cortex (2023-2025)"
+description = r"""The synaptic connectome of the neurons in a cubic millimeter of mouse visual cortex, from the IARPA MICrONS project: a volume spanning all cortical layers of the primary visual cortex (VISp) and the higher visual areas VISrl, VISal and VISlm of an adult mouse, imaged with serial-section transmission electron microscopy, automatically segmented, with synapses detected automatically, and partially proofread. Nodes are neurons, and a directed edge $(i, j)$ indicates chemical synapses from neuron $i$ (presynaptic) to neuron $j$ (postsynaptic).
+
+The networks are the three versions of the compilation of M. W. Reimann, which contains the synapses between the neurons with a cell body inside the larger of the two imaged subvolumes ("portion 65"), but no synapses from neurons outside it, named after the materialization (version) of the public MICrONS data, "minnie65_public", that they are based on:
+
+- `v1412`: version 3.0.0 of the compilation (August 2025), based on materialization v1412, with 50,948 neurons. It leaves out about 20,000 neurons near the periphery of the volume, whose connectivity is strongly affected by the boundary.
+- `v1181`: version 2.0.0 (September 2024), based on materialization v1181, with more proofread neurons and a newer cell type classification than v117 (71,981 neurons).
+- `v117`: version 1.0.0 (September 2023), based on materialization v117 (60,048 neurons), with the coarse cell classes of the Allen Institute.
+
+Each version has also a network with one edge per synapse (the "full" representation of the compilation), named `v1412_full`, `v1181_full` and `v117_full`, with the same nodes: two neurons are connected by as many parallel edges as there are synapses between them (11.6, 13.5 and 10.1 million edges).
+
+The version 3.0.0 has also a "distance-dependent control" (`v1412_control`): a random network generated by the author of the compilation as a null model, in which, for each pathway (combination of the layers and synapse classes of the presynaptic and postsynaptic neurons), the connection probability is an exponentially decaying function of the distance between the neurons fitted to the data. It contains only the 45,598 excitatory neurons of `v1412` (the inhibitory neurons are left out of the control), and is unweighted (one edge per connected pair).
+
+All versions leave out a few hundred cell bodies associated with several cell types (merges of several neurons). All connections are included, without threshold on the number of synapses (most connections in the cortex consist of one or two synapses). Note that only a small fraction of the neurons were proofread (see the properties `status_axon` and `status_dendrite`); the axons of the others are mostly incomplete, and their connections may contain errors of the automated segmentation.
+
+The networks have the following properties:
+
+- `root_id`: the neuron's identifier (root ID) in the MICrONS data of that version.
+- `cell_type`: the type of the neuron, predicted from features of its soma and nucleus: in `v1412` and `v1181`, the morphological type (m-type; e.g. `L2a`, `L4b`, `L5ET`, `L6tall-a` for excitatory neurons; `PTC`, `DTC`, `STC` and `ITC` for perisomatic-targeting, distal-targeting, sparsely-targeting and inhibitory-targeting inhibitory neurons); in `v117`, the coarse cell class (e.g. `23P`, `4P`, `5P_IT`, `6CT`, `BC`, `MC`).
+- `classification_system` (`v1412` and `v1181`): `excitatory_neuron` or `inhibitory_neuron`.
+- `synapse_class` (`v1412`): whether the neuron is excitatory (`EXC`) or inhibitory (`INH`).
+- `layer` (`v1412`): the cortical layer of the neuron (`2` to `6`), derived from its type; empty for inhibitory neurons.
+- `tentative_region` (`v117`): the visual area of the neuron (`visp`, `visrl`, `visal` or `vislm`), which the author of the compilation warns is only a rough estimate.
+- `status_axon`, `status_dendrite`: whether the neuron's axon and dendrite were proofread (`t`) or not (`f`), for the neurons in the proofreading table of MICrONS; in `v117`, the proofreading status of the public release of the time: `non` (not proofread), `clean` (false merges removed) or `extended` (also false splits corrected); empty for the neurons that are not in that table, which were not proofread.
+- `volume` (`v1412` and `v1181`): the volume of the neuron's nucleus, in µm³.
+- `x`, `y`, `z`: the position of the neuron's soma, in µm ($y$ increases with the cortical depth).
+- Edge `weight`: the number of synapses from $i$ to $j$ (not in the `_full` networks and in `v1412_control`).
+- Edge `total_size`, `mean_size` (`v1181` and `v117`): the sum and the mean of the sizes of these synapses, as given by the synapse table of MICrONS (in voxels).
+- Edge `synapse_id`, `size` (`_full` networks): the identifier of the synapse in the synapse table of MICrONS, and its size (in voxels).
+- Edge `syn_x`, `syn_y`, `syn_z`, `soma_distance` (`v1181_full` and `v117_full`): the position of the synapse, in µm, and its distance from the soma of the postsynaptic neuron, in µm.
+
+In `v1412_control`, the nodes have the properties `root_id`, `cell_type` (the m-type), `synapse_class` (`EXC`), `layer`, `volume` and `x`, `y`, `z`, as above.
+
+The compilation's "valid" flag of the neurons, the identifiers of their cell bodies (supervoxel and nucleus IDs), and, in `v117`, their degrees (which the networks give), are left out. In the `_full` networks, the offsets of the synapses from the soma of the postsynaptic neuron (which follow from their positions) are left out. In `v1412_control`, a second set of coordinates of the neurons (`X`, `Y`, `Z`, not documented) and a few columns that are empty for almost all neurons (morphologies, spines, in-degrees) are left out."""
+tags = ['Biological', 'Connectome', 'Weighted', 'Multigraph', 'Metadata']
+url = ['https://zenodo.org/records/16744240', 'https://zenodo.org/records/13849415',
+ 'https://zenodo.org/records/8364070',
+ 'https://www.microns-explorer.org/cortical-mm3']
+citation = [('The MICrONS Consortium, "Functional connectomics spanning multiple areas of mouse visual cortex." Nature 640, 435-447 (2025)', 'https://doi.org/10.1038/s41586-025-08790-w'),
+ ('M. W. Reimann, "Connectomics of (part of) the MICrONS mm3 dataset." Zenodo, version 3.0.0 (2025)', 'https://doi.org/10.5281/zenodo.16744240')]
+icon_hash = None
+upstream_license = ("CC BY 4.0", "https://creativecommons.org/licenses/by/4.0/")
+# The three versions of the compilation (Zenodo records), with the
+# materialization of the MICrONS data they are based on
+upstream_prefix = 'https://zenodo.org/records'
+files = [('16744240/files/microns_mm3_connectome_v1412.h5', 'v1412', None),
+ ('13849415/files/microns_mm3_connectome_v1181.h5', 'v1181', None),
+ ('8364070/files/microns_mm3_connectome.h5', 'v117', None),
+ # one edge per synapse ("full" representation)
+ ('16744240/files/microns_mm3_connectome_v1412.h5', 'v1412_full', None),
+ ('13849415/files/microns_mm3_connectome_v1181.h5', 'v1181_full', None),
+ ('8364070/files/microns_mm3_connectome.h5', 'v117_full', None),
+ # the distance-dependent control of v3.0.0
+ ('16744240/files/microns_mm3_v1412_distance_dependent_control.h5',
+ 'v1412_control', None)]
+
+def fetch_upstream(force=False):
+ return fetch_upstream_files(__name__.split(".")[-1], upstream_prefix, files,
+ force)
+
+# The edge indices of the "full" representation of v1412 are stored by
+# pandas as a block of Python objects, i.e. a pickle in an HDF5 variable-length
+# value, which the HDF5 library fails to read (it is larger than its cache
+# entries). It is read directly from the global heap of the file, and
+# unpickled allowing only the numpy array constructors.
+_pickle_allowed = {("numpy._core.multiarray", "_reconstruct"),
+ ("numpy.core.multiarray", "_reconstruct"),
+ ("numpy", "ndarray"), ("numpy", "dtype")}
+
+class _Unpickler(pickle.Unpickler):
+ def find_class(self, module, name):
+ if (module, name) in _pickle_allowed:
+ return getattr(importlib.import_module(module), name)
+ raise pickle.UnpicklingError(f"not allowed: {module}.{name}")
+
+def _read_object_block(fname, path):
+ with h5py.File(fname, "r") as h:
+ ds = h[path]
+ assert ds.shape == (1,) and ds.dtype == object
+ filt, raw = ds.id.read_direct_chunk((0,))
+ assert filt == 0
+ length, addr, index = struct.unpack("<IQI", raw[:16])
+ with open(fname, "rb") as f:
+ f.seek(addr)
+ head = f.read(16)
+ assert head[:4] == b"GCOL", head
+ data = f.read(struct.unpack("<Q", head[8:16])[0] - 16)
+ o = 0
+ while o + 16 <= len(data):
+ idx, ref, _, size = struct.unpack("<HHIQ", data[o:o + 16])
+ if idx == index:
+ assert size == length
+ return _Unpickler(io.BytesIO(data[o + 16:o + 16 + size])).load()
+ if idx == 0:
+ break
+ o += 16 + (size + 7) // 8 * 8
+ raise KeyError(f"heap object {index} not found")
+
+def _read_block_frame(h, path, fname):
+ """The columns of a pandas frame stored in HDF5 (fixed format), as a dict
+ of arrays, read with h5py."""
+ g = h[path]
+ cols = {}
+ for b in range(int(g.attrs["nblocks"])):
+ items = [x.decode() for x in g[f"block{b}_items"][()]]
+ ds = g[f"block{b}_values"]
+ if ds.dtype == object:
+ vals = np.asarray(_read_object_block(fname, ds.name), dtype="int64")
+ else:
+ vals = ds[()]
+ for i, c in enumerate(items):
+ cols[c] = vals[:, i]
+ return cols
+
+def read_full(fname):
+ """The edges of the "full" representation (one per synapse), and their
+ properties (name: (type, values))."""
+ base = "/connectivity/full"
+ with h5py.File(fname, "r") as h:
+ ei = _read_block_frame(h, f"{base}/edge_indices", fname)
+ ew = _read_block_frame(h, f"{base}/edges", fname)
+ edges = np.column_stack((ei["row"], ei["col"]))
+ del ei
+ props = {"synapse_id": ("int64_t", ew["id"]), "size": ("int", ew["size"])}
+ if "x_nm" in ew:
+ for c in "xyz":
+ props[f"syn_{c}"] = ("double", ew[f"{c}_nm"] / 1000)
+ props["soma_distance"] = ("double", ew["soma_offset_um"])
+ return edges, props
+
+@cache_network()
+@coerce_props(vforce={"layer": "string"})
+@annotate()
+def parse(alts=None):
+ name = __name__.split(".")[-1]
+ for fname, alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ # A pandas HDFStore in the format of Connectome-utilities; only its
+ # "condensed" representation is read (one edge per connected pair,
+ # with the number of synapses), with h5py (the HDF5 library fails on
+ # the edge table of the "full" representation)
+ with open_upstream_file(name, fname, "rb", seekable=True) as f:
+ data = f.read()
+ os.makedirs("cache/temp", exist_ok=True)
+ with tempfile.NamedTemporaryFile(dir="cache/temp", suffix=".h5") as tmp:
+ tmp.write(data)
+ tmp.flush()
+ del data
+ with pd.HDFStore(tmp.name, "r") as st:
+ if alt.endswith("_control"):
+ base = "/connectivity/full_matrix"
+ vp = st[f"{base}/vertex_properties"]
+ vp = vp.rename(columns={"mtype": "cell_type",
+ "source__volume": "volume"})
+ vp[["x_nm", "y_nm", "z_nm"]] = vp[["x", "y", "z"]] * 1000
+ ei = st[f"{base}/edge_indices"]
+ ew = st[f"{base}/edges"]
+ assert (ew["data"] == 1).all()
+ ew = ew.drop(columns=["data"])
+ else:
+ base = "/connectivity/condensed"
+ vp = st[f"{base}/vertex_properties"]
+ if alt.endswith("_full"):
+ ei = ew = None
+ else:
+ ei = st[f"{base}/edge_indices"]
+ ew = st[f"{base}/edges"]
+ if alt.endswith("_full"):
+ full_edges, full_props = read_full(tmp.name)
+
+ g = Graph()
+ g.add_vertex(len(vp))
+ g.vp.root_id = g.new_vp("int64_t", vals=vp["pt_root_id"].to_numpy())
+ def strings(c, empty=()):
+ x = vp[c].astype(object).fillna("").astype(str)
+ return x.where(~x.isin(empty), "").tolist()
+ g.vp.cell_type = g.new_vp("string", vals=strings("cell_type"))
+ for c in ["classification_system", "synapse_class", "tentative_region"]:
+ if c in vp:
+ g.vp[c] = g.new_vp("string", vals=strings(c))
+ if "layer" in vp:
+ g.vp.layer = g.new_vp("string", vals=strings("layer", ["N/A"]))
+ for s in ["status_axon", "status_dendrite"]:
+ if s in vp:
+ g.vp[s] = g.new_vp("string", vals=strings(s, ["empty"]))
+ if "volume" in vp:
+ g.vp.volume = g.new_vp("double", vals=vp["volume"].to_numpy())
+ for c in "xyz":
+ g.vp[c] = g.new_vp("double", vals=vp[f"{c}_nm"].to_numpy() / 1000)
+
+ if alt.endswith("_full"):
+ assert full_edges.min() >= 0 and full_edges.max() < len(vp)
+ g.add_edge_list(full_edges)
+ del full_edges
+ for k, (t, vals) in full_props.items():
+ g.ep[k] = g.new_ep(t, vals=vals)
+ del full_props
+ yield alt, g
+ continue
+ g.add_edge_list(ei[["row", "col"]].to_numpy())
+ if "count" in ew:
+ g.ep.weight = g.new_ep("int", vals=ew["count"].to_numpy())
+ for c in ["total_size", "mean_size"]:
+ if c in ew:
+ g.ep[c] = g.new_ep("double", vals=ew[c].to_numpy())
+ yield alt, g