History of hosts_lepidoptera
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] Add the date when each entry was added, and remove icon_hash beff7c9
The dates are those of the commits that first added the entries' files (2020-08-11 for the entries of the initial commit); renamed entries keep the date of their first name. Changed: none
diff --git a/entries/hosts_lepidoptera/__init__.py b/entries/hosts_lepidoptera/__init__.py
index f49794e..03b078e 100644
--- a/entries/hosts_lepidoptera/__init__.py
+++ b/entries/hosts_lepidoptera/__init__.py
@@ -41,7 +41,7 @@ tags = ['Biological', 'Ecological', 'Food web', 'Unweighted', 'Bipartite', 'Mult
url = ['https://doi.org/10.5519/havt50xw', 'https://data.nhm.ac.uk/dataset/hosts']
citation = [('G. S. Robinson, P. R. Ackery, I. Kitching, G. W. Beccaloni, L. M. Hernández, "HOSTS - a Database of the World\'s Lepidopteran Hostplants." Natural History Museum (2023)',
'https://doi.org/10.5519/havt50xw')]
-icon_hash = None
+date_added = '2026-10-31'
upstream_license = ('Creative Commons CC0 1.0 Universal (CC0 1.0)',
'https://creativecommons.org/publicdomain/zero/1.0/')
# The CSV download of the data portal is behind a bot challenge; the records
[entries] Prefix the properties in the descriptions with (Node), (Edge) or (Graph) bc3dbbd
The lists of properties at the end of the descriptions marked edge properties with "Edge" but left node properties unmarked, or grouped them in "Nodes: `a`, ...; `b`, ..." items, so the kind of a property was often unclear. Every item now starts with (Node), (Edge) or (Graph), followed by the property names, a colon and the explanation, one item per property or group of properties; qualifications (kinds of nodes, networks) follow the names in parentheses. Properties that were only named got a short explanation, and a few items that named the wrong kind of property were corrected.
diff --git a/entries/hosts_lepidoptera/__init__.py b/entries/hosts_lepidoptera/__init__.py
index 95597fc..f49794e 100644
--- a/entries/hosts_lepidoptera/__init__.py
+++ b/entries/hosts_lepidoptera/__init__.py
@@ -29,14 +29,14 @@ Nodes are lepidopteran taxa (species or subspecies, or genera when the species i
The network has the following properties:
-- `is_lepidoptera`: whether the node is a lepidopteran (otherwise it is a hostplant).
-- `name`: the name of the taxon (genus, species and subspecies or variety).
-- `family`, `genus`, `species`, `subspecies`: the family and the parts of the name of the taxon.
-- `author` (lepidoptera): the author of the name of the lepidopteran taxon.
-- Edge `hosts_id`: the identifier of the record.
-- Edge `location`: the country or biogeographic region of the record.
-- Edge `damage`: the parts of the plant fed upon by the caterpillar, and how (e.g. `on leaves`, `in fruits`, `in stems`; empty if not given).
-- Edge `lab_rearing`: `Y` if the record comes from rearing in a laboratory (`?` if this is uncertain; empty otherwise)."""
+- (Node) `is_lepidoptera`: whether the node is a lepidopteran (otherwise it is a hostplant).
+- (Node) `name`: the name of the taxon (genus, species and subspecies or variety).
+- (Node) `family`, `genus`, `species`, `subspecies`: the family and the parts of the name of the taxon.
+- (Node) `author` (lepidoptera): the author of the name of the lepidopteran taxon.
+- (Edge) `hosts_id`: the identifier of the record.
+- (Edge) `location`: the country or biogeographic region of the record.
+- (Edge) `damage`: the parts of the plant fed upon by the caterpillar, and how (e.g. `on leaves`, `in fruits`, `in stems`; empty if not given).
+- (Edge) `lab_rearing`: `Y` if the record comes from rearing in a laboratory (`?` if this is uncertain; empty otherwise)."""
tags = ['Biological', 'Ecological', 'Food web', 'Unweighted', 'Bipartite', 'Multigraph', 'Metadata']
url = ['https://doi.org/10.5519/havt50xw', 'https://data.nhm.ac.uk/dataset/hosts']
citation = [('G. S. Robinson, P. R. Ackery, I. Kitching, G. W. Beccaloni, L. M. Hernández, "HOSTS - a Database of the World\'s Lepidopteran Hostplants." Natural History Museum (2023)',
[entries] New entry: hosts_lepidoptera ee6d985
diff --git a/entries/hosts_lepidoptera/__init__.py b/entries/hosts_lepidoptera/__init__.py
new file mode 100644
index 0000000..95597fc
--- /dev/null
+++ b/entries/hosts_lepidoptera/__init__.py
@@ -0,0 +1,130 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import urllib.parse
+from collections import defaultdict
+
+from .. import *
+
+title = "HOSTS database of lepidopteran hostplants (2023)"
+description = r"""Bipartite network of butterflies and moths (Lepidoptera) and the plants their caterpillars feed on, from HOSTS, the database of the world's lepidopteran hostplants of the Natural History Museum, London, compiled by G. S. Robinson and collaborators from the published literature and from museum collections. The database is no longer updated; the data are those published on the NHM Data Portal in 2023.
+
+Nodes are lepidopteran taxa (species or subspecies, or genera when the species is not given) and hostplant taxa (species, subspecies or varieties, or genera when the species of the plant is not given), and an edge connects a lepidopteran to a hostplant for each record of the database, i.e. each location (a country, or a biogeographic region) in which the caterpillar was recorded on the plant, so that a pair can be connected by several edges. Taxa are identified by their names as given in the database (genus, species and subspecies).
+
+The network has the following properties:
+
+- `is_lepidoptera`: whether the node is a lepidopteran (otherwise it is a hostplant).
+- `name`: the name of the taxon (genus, species and subspecies or variety).
+- `family`, `genus`, `species`, `subspecies`: the family and the parts of the name of the taxon.
+- `author` (lepidoptera): the author of the name of the lepidopteran taxon.
+- Edge `hosts_id`: the identifier of the record.
+- Edge `location`: the country or biogeographic region of the record.
+- Edge `damage`: the parts of the plant fed upon by the caterpillar, and how (e.g. `on leaves`, `in fruits`, `in stems`; empty if not given).
+- Edge `lab_rearing`: `Y` if the record comes from rearing in a laboratory (`?` if this is uncertain; empty otherwise)."""
+tags = ['Biological', 'Ecological', 'Food web', 'Unweighted', 'Bipartite', 'Multigraph', 'Metadata']
+url = ['https://doi.org/10.5519/havt50xw', 'https://data.nhm.ac.uk/dataset/hosts']
+citation = [('G. S. Robinson, P. R. Ackery, I. Kitching, G. W. Beccaloni, L. M. Hernández, "HOSTS - a Database of the World\'s Lepidopteran Hostplants." Natural History Museum (2023)',
+ 'https://doi.org/10.5519/havt50xw')]
+icon_hash = None
+upstream_license = ('Creative Commons CC0 1.0 Universal (CC0 1.0)',
+ 'https://creativecommons.org/publicdomain/zero/1.0/')
+# The CSV download of the data portal is behind a bot challenge; the records
+# are read from the datastore API of the portal (CKAN), 1,000 at a time,
+# and stored as JSON lines
+API = "https://data.nhm.ac.uk/api/3/action/datastore_search"
+RESOURCE = "877f387a-36a3-486c-a0c1-b8d5fb69f85a"
+upstream_prefix = None
+FNAME = "hosts-records.jsonl"
+files = [(FNAME, None, None)]
+
+def fetch_upstream(force=False):
+ name = __name__.split(".")[-1]
+ if not force and upstream_file_exists(name, FNAME):
+ return
+ url = f"{API}?resource_id={RESOURCE}&limit=1000"
+ records = []
+ total = None
+ after = None
+ while True:
+ u = url if after is None else \
+ f"{url}&after={urllib.parse.quote(json.dumps(after))}"
+ with open_url(u) as f:
+ res = json.load(f)["result"]
+ if total is None:
+ total = res["total"]
+ records.extend(res["records"])
+ after = res.get("after")
+ if not res["records"] or after is None or len(records) >= total:
+ break
+ if len(records) != total:
+ raise FetchError(f"{name}: fetched {len(records)} of {total} records")
+ with open_upstream_file(name, FNAME, "wb") as out:
+ for r in records:
+ out.write((json.dumps(r, ensure_ascii=False) + "\n").encode("utf8"))
+
+INSECT = ["Insect Family", "Insect Genus", "Insect Species", "Insect Subspecies",
+ "Insect Author"]
+PLANT = ["Hostplant Family", "Hostplant Genus", "Hostplant Species",
+ "Hostplant Subspecies/var"]
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+ name = __name__.split(".")[-1]
+ with open_upstream_file(name, FNAME, "r") as f:
+ records = [json.loads(l) for l in f]
+ records.sort(key=lambda r: int(r["_id"]))
+ vidx = {}
+ vvals = defaultdict(list)
+ def add_vertex(is_l, fam, gen, sp, ssp, author=""):
+ key = (is_l, gen, sp, ssp) if is_l else (is_l, fam, gen, sp, ssp)
+ v = vidx.get(key)
+ if v is None:
+ v = vidx[key] = len(vidx)
+ vvals["is_lepidoptera"].append(is_l)
+ vvals["name"].append(" ".join(x for x in (gen, sp, ssp) if x))
+ vvals["family"].append(fam)
+ vvals["genus"].append(gen)
+ vvals["species"].append(sp)
+ vvals["subspecies"].append(ssp)
+ vvals["author"].append(author)
+ return v
+ edges = []
+ evals = defaultdict(list)
+ for r in records:
+ r = {k: (v or "").strip() if isinstance(v, str) or v is None else v
+ for k, v in r.items()}
+ u = add_vertex(True, *[r[c] for c in INSECT])
+ v = add_vertex(False, *[r[c] for c in PLANT])
+ edges.append((u, v))
+ evals["hosts_id"].append(int(r["HOSTS ID"]))
+ evals["location"].append(r["Location"])
+ evals["damage"].append(r["Damage"])
+ evals["lab_rearing"].append(r["Lab Rearing"])
+ g = Graph(directed=False)
+ g.add_vertex(len(vidx))
+ g.add_edge_list(numpy.array(edges, dtype="int64").reshape(-1, 2))
+ g.vp.is_lepidoptera = g.new_vp("bool", vals=vvals["is_lepidoptera"])
+ for p in ["name", "family", "genus", "species", "subspecies", "author"]:
+ g.vp[p] = g.new_vp("string", vals=vvals[p])
+ g.ep.hosts_id = g.new_ep("int64_t", vals=evals["hosts_id"])
+ for p in ["location", "damage", "lab_rearing"]:
+ g.ep[p] = g.new_ep("string", vals=evals[p])
+ yield None, g