History of fly_larva
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] Add the date when each entry was added, and remove icon_hash beff7c9
The dates are those of the commits that first added the entries' files (2020-08-11 for the entries of the initial commit); renamed entries keep the date of their first name. Changed: none
diff --git a/entries/fly_larva/__init__.py b/entries/fly_larva/__init__.py
index 2827621..ee49b86 100644
--- a/entries/fly_larva/__init__.py
+++ b/entries/fly_larva/__init__.py
@@ -26,7 +26,7 @@ tags = ['Biological', 'Connectome', 'Weighted', 'Multigraph', 'Metadata']
url = ['https://doi.org/10.1126/science.add9330',
'https://europepmc.org/article/PMC/PMC7614541']
citation = [('M. Winding et al., "The connectome of an insect brain", Science 379 eadd9330 (2023)', 'https://doi.org/10.1126/science.add9330')]
-icon_hash = None
+date_added = '2023-03-11'
upstream_license = None
# The publisher's file server refuses automated downloads, so the
# supplementary files (the same as the publisher's Data S1 and S2) are
[entries] Remove unneeded code from the entries 2c69f19
Unneeded "global files" statements, unused imports and variables, f-strings without placeholders, duplicate keys in foodweb_little_rock (keeping the order and the values used), and the second definition of get_lines() in twitter_higgs. Nothing changes in the networks or descriptions. Changed: none
diff --git a/entries/fly_larva/__init__.py b/entries/fly_larva/__init__.py
index 3f56f01..2827621 100644
--- a/entries/fly_larva/__init__.py
+++ b/entries/fly_larva/__init__.py
@@ -43,7 +43,6 @@ def fetch_upstream(force=False):
@coerce_props()
@annotate()
def parse(alts=None):
- global files
name = __name__.split(".")[-1]
for fnames, alt, fmt in files:
if alts is not None and alt not in alts:
[entries] Move unavailable upstream sources to verified alternatives 2137af6
The upstream sources of these entries are gone, moved, or refuse automated downloads; each now downloads the same data (byte-identical to the cached files, or for board_directors, identical apart from line endings) from: - add_health: the Wayback Machine (Moreno's site, as its sibling entries) - board_directors: T. Opsahl's site (opsahl.co.uk) - copenhagen, cow_signed_relations, fullerene_structures, german_power_grid: figshare's API (its downloader refuses the server) - drosophila_flybi: the same address, without a doubled slash - dutch_criticism: V. Batagelj's repository of Pajek data on GitHub - fly_larva: the copy of the supplementary files in Europe PMC - foursquare: the authors' Google Drive file - game_thrones, microsoft_concept, un_migrations: pinned Wayback Machine snapshots - internet_top_pop: the Topology Zoo repository on GitHub - scotus_majority: Harvard Dataverse and Wayback Machine snapshots
diff --git a/entries/fly_larva/__init__.py b/entries/fly_larva/__init__.py
index 16dc6e3..3f56f01 100644
--- a/entries/fly_larva/__init__.py
+++ b/entries/fly_larva/__init__.py
@@ -23,13 +23,17 @@ import io
title = "Drosophila larva brain (2023)"
description = """A complete synaptic map of the brain connectome of the larva of the fruit fly Drosophila melanogaster. Nodes are neurons, and edges are synaptic connections, traced individually from brain image sections using three-dimensional electron microscopy–based reconstruction. A directed edge points from the presynaptic to the postsynaptic neuron (from the rows to the columns of the published connectivity matrices), with a separate edge for each type of connection between the pair, according to the presynaptic and postsynaptic compartments (`'ad'`: axo-dendritic, `'aa'`: axo-axonic, `'dd'`: dendro-dendritic, `'da'`: dendro-axonic), so that a pair of neurons can be connected by several parallel edges; each edge is weighted by its number of synapses. Node metadata include the neuron hemisphere, hemispherical homologue, cell type, annotations, and inferred cluster. Edge metadata include the type of interaction (`'aa'`, `'ad'`, `'da'`, `'dd'`), and synapse count."""
tags = ['Biological', 'Connectome', 'Weighted', 'Multigraph', 'Metadata']
-url = 'https://doi.org/10.1126/science.add9330'
+url = ['https://doi.org/10.1126/science.add9330',
+ 'https://europepmc.org/article/PMC/PMC7614541']
citation = [('M. Winding et al., "The connectome of an insect brain", Science 379 eadd9330 (2023)', 'https://doi.org/10.1126/science.add9330')]
icon_hash = None
upstream_license = None
-upstream_prefix = 'https://www.science.org/doi/suppl/10.1126/science.add9330/suppl_file/'
-files = [(('science.add9330_data_s1_to_s4.zip:science.add9330_data_s1.zip',
- 'science.add9330_data_s1_to_s4.zip:science.add9330_data_s2.csv'), None, 'csv')]
+# The publisher's file server refuses automated downloads, so the
+# supplementary files (the same as the publisher's Data S1 and S2) are
+# obtained from the Europe PMC web service (as a zip archive).
+upstream_prefix = 'https://www.ebi.ac.uk/europepmc/webservices/rest/PMC7614541'
+files = [(('supplementaryFiles:zip:EMS175448-supplement-Supplementary_Data_S1.zip',
+ 'supplementaryFiles:zip:EMS175448-supplement-Supplementary_Data_S2.csv'), None, 'csv')]
def fetch_upstream(force=False):
return fetch_upstream_files(__name__.split(".")[-1], upstream_prefix, files,
[entries] Fix parse bugs and improve descriptions and metadata 6bb6851
- parse fixes from the review of the descriptions, e.g. us_patents (directed), celegans_2019 (synapse direction), ecoli_transcription and yeast_transcription (regulator to target), bitcoin (direction, phantom node), ego_social (Google+ features and circles), scotus_majority, arxiv_authors and social_location (as in SNAP), caida_as, dnc, dom, mist, reuters and bibsonomy (names on the right nodes), uci_zoo - every description now states explicitly what the nodes and edges represent; corrected titles, years, tags, citations and licences - the "Hypergraph" tag is removed; very large entries are marked `huge = True` (see --skip-huge) - ICON ids of partial duplicates added (hiv_transmission, congress_bills, openflights, dutch_school)
diff --git a/entries/fly_larva/__init__.py b/entries/fly_larva/__init__.py
index 3e180f5..16dc6e3 100644
--- a/entries/fly_larva/__init__.py
+++ b/entries/fly_larva/__init__.py
@@ -21,8 +21,8 @@ from zipfile import ZipFile
import io
title = "Drosophila larva brain (2023)"
-description = """A complete synaptic map of the brain connectome of the larva of the fruit fly Drosophila melanogaster. Nodes are neurons, and edges are synaptic connections, traced individually from brain image sections using three-dimensional electron microscopy–based reconstruction. Node metadata include the neuron hempisphere, hemispherical homologue, cell type, annotations, and inferred cluster. Edge metadata include the type of interaction (`'aa'`, `'ad'`, `'da'`, `'dd'`), and synapse count."""
-tags = ['Biological', 'Connectome', 'Weighted', 'Metadata']
+description = """A complete synaptic map of the brain connectome of the larva of the fruit fly Drosophila melanogaster. Nodes are neurons, and edges are synaptic connections, traced individually from brain image sections using three-dimensional electron microscopy–based reconstruction. A directed edge points from the presynaptic to the postsynaptic neuron (from the rows to the columns of the published connectivity matrices), with a separate edge for each type of connection between the pair, according to the presynaptic and postsynaptic compartments (`'ad'`: axo-dendritic, `'aa'`: axo-axonic, `'dd'`: dendro-dendritic, `'da'`: dendro-axonic), so that a pair of neurons can be connected by several parallel edges; each edge is weighted by its number of synapses. Node metadata include the neuron hemisphere, hemispherical homologue, cell type, annotations, and inferred cluster. Edge metadata include the type of interaction (`'aa'`, `'ad'`, `'da'`, `'dd'`), and synapse count."""
+tags = ['Biological', 'Connectome', 'Weighted', 'Multigraph', 'Metadata']
url = 'https://doi.org/10.1126/science.add9330'
citation = [('M. Winding et al., "The connectome of an insect brain", Science 379 eadd9330 (2023)', 'https://doi.org/10.1126/science.add9330')]
icon_hash = None
[entries] Fix parse bugs and improve the metadata of existing entries c996a0e
From a review of the dataset descriptions against the upstream sources and papers, and from user reports: - parse fixes (wrong edge direction or directedness, shifted or misnamed properties, merged node identities, duplicated or missing edges and nodes, wrong types), e.g. arxiv_citation (the HepTh dates were a copy of the HepPh ones; now from the KDD Cup 2003 data), eu_procurements, foursquare_global, hiv_transmission, genetic_multiplex, gnutella (adds the 05 snapshot), roadnet, macaque_neural, python_dependency - descriptions rewritten from the upstream sources, documenting every property - corrected titles, citations and DOIs, licences (ustream_license typo, the data's own terms), tags and ICON ids - url lists for entries with several sources or dataset pages - SocioPatterns entries moved to the site's new data location, with the daily networks of sp_infectious and sp_primary_school - the Michael Fire entries marked as orphans (served from the mirror) - redundant type forcings removed after the graph-tool coercion fixes
diff --git a/entries/fly_larva/__init__.py b/entries/fly_larva/__init__.py
index 7e6e419..3e180f5 100644
--- a/entries/fly_larva/__init__.py
+++ b/entries/fly_larva/__init__.py
@@ -26,7 +26,7 @@ tags = ['Biological', 'Connectome', 'Weighted', 'Metadata']
url = 'https://doi.org/10.1126/science.add9330'
citation = [('M. Winding et al., "The connectome of an insect brain", Science 379 eadd9330 (2023)', 'https://doi.org/10.1126/science.add9330')]
icon_hash = None
-ustream_license = None
+upstream_license = None
upstream_prefix = 'https://www.science.org/doi/suppl/10.1126/science.add9330/suppl_file/'
files = [(('science.add9330_data_s1_to_s4.zip:science.add9330_data_s1.zip',
'science.add9330_data_s1_to_s4.zip:science.add9330_data_s2.csv'), None, 'csv')]
Update copyright information 73a4f0d
diff --git a/entries/fly_larva/__init__.py b/entries/fly_larva/__init__.py
index c063924..7e6e419 100644
--- a/entries/fly_larva/__init__.py
+++ b/entries/fly_larva/__init__.py
@@ -1,7 +1,7 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
-# Copyright (C) 2020-2024 Tiago de Paula Peixoto <tiago@skewed.de>
+# Copyright (C) 2020-2026 Tiago de Paula Peixoto <tiago@skewed.de>
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
Update copyright information 3d288ea
diff --git a/entries/fly_larva/__init__.py b/entries/fly_larva/__init__.py
index b4c9e38..c063924 100644
--- a/entries/fly_larva/__init__.py
+++ b/entries/fly_larva/__init__.py
@@ -1,7 +1,7 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
-# Copyright (C) 2020-2023 Tiago de Paula Peixoto <tiago@skewed.de>
+# Copyright (C) 2020-2024 Tiago de Paula Peixoto <tiago@skewed.de>
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
Add 'fly_larva' entry 28d4425
diff --git a/entries/fly_larva/__init__.py b/entries/fly_larva/__init__.py
new file mode 100644
index 0000000..b4c9e38
--- /dev/null
+++ b/entries/fly_larva/__init__.py
@@ -0,0 +1,106 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2023 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from .. import *
+from zipfile import ZipFile
+import io
+
+title = "Drosophila larva brain (2023)"
+description = """A complete synaptic map of the brain connectome of the larva of the fruit fly Drosophila melanogaster. Nodes are neurons, and edges are synaptic connections, traced individually from brain image sections using three-dimensional electron microscopy–based reconstruction. Node metadata include the neuron hempisphere, hemispherical homologue, cell type, annotations, and inferred cluster. Edge metadata include the type of interaction (`'aa'`, `'ad'`, `'da'`, `'dd'`), and synapse count."""
+tags = ['Biological', 'Connectome', 'Weighted', 'Metadata']
+url = 'https://doi.org/10.1126/science.add9330'
+citation = [('M. Winding et al., "The connectome of an insect brain", Science 379 eadd9330 (2023)', 'https://doi.org/10.1126/science.add9330')]
+icon_hash = None
+ustream_license = None
+upstream_prefix = 'https://www.science.org/doi/suppl/10.1126/science.add9330/suppl_file/'
+files = [(('science.add9330_data_s1_to_s4.zip:science.add9330_data_s1.zip',
+ 'science.add9330_data_s1_to_s4.zip:science.add9330_data_s2.csv'), None, 'csv')]
+
+def fetch_upstream(force=False):
+ return fetch_upstream_files(__name__.split(".")[-1], upstream_prefix, files,
+ force)
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+ global files
+ name = __name__.split(".")[-1]
+ for fnames, alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ if isinstance(fnames, str):
+ fnames = [fnames]
+ with ExitStack() as stack:
+ fs = [stack.enter_context(open_upstream_file(name, fn, "rb")) for fn in fnames]
+ g = Graph()
+ g.ep.count = g.new_ep("int")
+ g.ep.etype = g.new_ep("string")
+ g.vp.vids = g.new_vp("int")
+ vertices = {}
+ def get_vertex(vid, add_new=True):
+ vid = int(vid)
+ v = vertices.get(vid, None)
+ if v is None and add_new:
+ v = g.add_vertex()
+ g.vp.vids[v] = vid
+ vertices[vid] = v
+ if v is None:
+ raise KeyError("vertex not found:", vid)
+ return v
+ with ZipFile(fs[0]) as zf:
+ for et in ["da", "dd", "aa", "ad"]:
+ with zf.open(f'Supplementary-Data-S1/{et}_connectivity_matrix.csv') as fb:
+ with io.TextIOWrapper(fb) as f:
+ for i, line in enumerate(f):
+ if i == 0:
+ ids = line.strip().split(",")[1:]
+ continue
+ vals = line.strip().split(",")
+ src = get_vertex(vals[0])
+ for j, w in enumerate(vals[1:]):
+ w = float(w)
+ if w == 0:
+ continue
+ tgt = ids[j]
+ e = g.add_edge(src, get_vertex(tgt))
+ g.ep.count[e] = w
+ g.ep.etype[e] = et
+ g.vp.hemisphere = g.new_vp("string")
+ g.vp.homologue = g.new_vp("int", val=-1)
+ g.vp.cell_type = g.new_vp("string")
+ g.vp.annotations = g.new_vp("string")
+ g.vp.cluster = g.new_vp("int")
+ with io.TextIOWrapper(fs[1]) as f:
+ for i, line in enumerate(f):
+ if i == 0:
+ continue
+ left, right, celltype, annotations, cluster = line.strip().split(",")
+ if cluster == "no cluster":
+ cluster = -1
+ for vid in [left, right]:
+ if vid != "no pair":
+ v = get_vertex(vid, True)
+ g.vp.cell_type[v] = celltype
+ g.vp.annotations[v] = annotations
+ g.vp.cluster[v] = cluster
+ g.vp.hemisphere[v] = "left" if vid == left else "right"
+ if "no pair" not in [left, right]:
+ g.vp.homologue[vertices[int(left)]] = int(right)
+ g.vp.homologue[vertices[int(right)]] = int(left)
+ yield alt, g