History of us_transmission_lines
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] Add the date when each entry was added, and remove icon_hash beff7c9
The dates are those of the commits that first added the entries' files (2020-08-11 for the entries of the initial commit); renamed entries keep the date of their first name. Changed: none
diff --git a/entries/us_transmission_lines/__init__.py b/entries/us_transmission_lines/__init__.py
index c9c09a1..d2f63a0 100644
--- a/entries/us_transmission_lines/__init__.py
+++ b/entries/us_transmission_lines/__init__.py
@@ -56,7 +56,7 @@ citation = [('Homeland Infrastructure Foundation-Level Data (HIFLD), "U.S. Elect
# Agreement as its terms of use
upstream_license = ("US Government work (public domain); Esri Master License Agreement (ArcGIS Online copy)",
"https://www.esri.com/en-us/legal/terms/full-master-agreement")
-icon_hash = None
+date_added = '2026-10-31'
# The layer of the Homeland Infrastructure Foundation-Level Data (HIFLD)
# Open portal, which was shut down in 2025, as archived by Esri's Federal
[entries] Prefix the properties in the descriptions with (Node), (Edge) or (Graph) bc3dbbd
The lists of properties at the end of the descriptions marked edge properties with "Edge" but left node properties unmarked, or grouped them in "Nodes: `a`, ...; `b`, ..." items, so the kind of a property was often unclear. Every item now starts with (Node), (Edge) or (Graph), followed by the property names, a colon and the explanation, one item per property or group of properties; qualifications (kinds of nodes, networks) follow the names in parentheses. Properties that were only named got a short explanation, and a few items that named the wrong kind of property were corrected.
diff --git a/entries/us_transmission_lines/__init__.py b/entries/us_transmission_lines/__init__.py
index df16047..c9c09a1 100644
--- a/entries/us_transmission_lines/__init__.py
+++ b/entries/us_transmission_lines/__init__.py
@@ -33,8 +33,21 @@ The networks are undirected. The geometries of the lines (their courses as polyl
The network has the following properties:
-- Nodes: in `endpoints`, `latitude` and `longitude`, the position of the end point; in `substations`, `name`, the name of the substation (empty if not available).
-- Edges: `hifld_id`, the id of the line in HIFLD; `type`, the type of line (e.g. `AC; OVERHEAD`, `AC; UNDERGROUND`, `DC; OVERHEAD`); `status` (`IN SERVICE`, `INACTIVE`, `UNDER CONSTRUCTION`); `owner`, the owner of the line; `voltage`, its voltage (kV; NaN if not available); `volt_class`, the voltage class (e.g. `100-161`, `345`, `DC`); `inferred`, whether the voltage was inferred (`Y` or `N`); `sub_1` and `sub_2`, the names of the substations at the ends of the line, as given in the data; `source`, the sources of the data; `source_date`, the date of the source (Unix time; -1 where missing); `val_method`, the method of validation (e.g. `IMAGERY`); `val_date`, the date of validation (Unix time; -1 where missing); `length`, the length of the line along its course (km, computed from its geometry). Values that are not available are empty strings."""
+- (Node) `latitude`, `longitude` (`endpoints`): the position of the end point.
+- (Node) `name` (`substations`): the name of the substation (empty if not available).
+- (Edge) `hifld_id`: the id of the line in HIFLD.
+- (Edge) `type`: the type of line (e.g. `AC; OVERHEAD`, `AC; UNDERGROUND`, `DC; OVERHEAD`).
+- (Edge) `status`: the status of the line (`IN SERVICE`, `INACTIVE`, `UNDER CONSTRUCTION`).
+- (Edge) `owner`: the owner of the line.
+- (Edge) `voltage`: its voltage (kV; NaN if not available).
+- (Edge) `volt_class`: the voltage class (e.g. `100-161`, `345`, `DC`).
+- (Edge) `inferred`: whether the voltage was inferred (`Y` or `N`).
+- (Edge) `sub_1` and `sub_2`: the names of the substations at the ends of the line, as given in the data.
+- (Edge) `source`: the sources of the data.
+- (Edge) `source_date`: the date of the source (Unix time; -1 where missing).
+- (Edge) `val_method`: the method of validation (e.g. `IMAGERY`).
+- (Edge) `val_date`: the date of validation (Unix time; -1 where missing).
+- (Edge) `length`: the length of the line along its course (km, computed from its geometry). Values that are not available are empty strings."""
tags = ['Technological', 'Power grid', 'Spatial', 'Weighted', 'Multigraph', 'Metadata']
url = ['https://www.arcgis.com/home/item.html?id=d4090758322c4d32a4cd002ffaa0aa12']
citation = [('Homeland Infrastructure Foundation-Level Data (HIFLD), "U.S. Electric Power Transmission Lines." US Department of Homeland Security, dataset, archived on ArcGIS Online (2024)', 'https://www.arcgis.com/home/item.html?id=d4090758322c4d32a4cd002ffaa0aa12')]
[entries] New entry: us_transmission_lines bc549b1
diff --git a/entries/us_transmission_lines/__init__.py b/entries/us_transmission_lines/__init__.py
new file mode 100644
index 0000000..df16047
--- /dev/null
+++ b/entries/us_transmission_lines/__init__.py
@@ -0,0 +1,190 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from .. import *
+import json
+import urllib.parse
+import process_entry
+
+title = "US electric power transmission lines (HIFLD, 2024)"
+description = """Network of the electric power transmission lines of the United States (including Alaska, Hawaii, Puerto Rico and Guam), from the "Electric Power Transmission Lines" layer of the Homeland Infrastructure Foundation-Level Data (HIFLD) Open portal of the US Department of Homeland Security, in its last version (data current as of September 30, 2024). The layer was compiled from aerial imagery, the forms EIA-860 and EIA-861 of the US Energy Information Administration, OpenStreetMap and other public sources, and covers lines of 69 kV up to 765 kV, and some of lower voltage; underground lines are included where sources were available. The HIFLD Open portal was shut down in 2025; the data are taken from the copy archived (and no longer updated) on ArcGIS Online by Esri's Federal User Community.
+
+In both networks each edge is a transmission line (94,619 in all). The layer gives the lines as geometries, with the names of the substations at their ends (`SUB_1` and `SUB_2`) as attributes; most of these names are placeholders (e.g. "UNKNOWN128553" or "TAP139917"), and they are not always consistent between lines that meet at the same point, so the topology is given in two ways:
+
+- `endpoints`: the nodes are the end points of the lines (substations, taps and junctions), built from the geometries: two lines are joined at a node if they end at the same point (after rounding the coordinates to $10^{-7}$ degrees, about 1 cm). Lines that cross or touch each other elsewhere than at their end points are not joined. The network has parallel edges (4,502, e.g. circuits running side by side) and 24 self-loops (lines ending where they start); it is not connected (836 components, the largest with 72,103 nodes).
+- `substations`: the nodes are the substation names of the lines' ends, as given in the data (including the numbered placeholders, which identify taps, risers, dead ends and unknown substations); two lines are joined if they name the same substation. Names are not unique across the country (e.g. 96 line ends name a substation "MIDWAY", from Florida to Texas and California), so distinct substations of the same name are merged. Where a name is not available ("NOT AVAILABLE", 17,559 line ends), the end gets a node of its own, with an empty name. The network has parallel edges (3,532) and 186 self-loops (lines that name the same substation at both ends); it is not connected (5,610 components, the largest with 65,817 nodes).
+
+The networks are undirected. The geometries of the lines (their courses as polylines) are not included, nor the constant attributes (the NAICS code of the industry), the layer's internal object ids, and its lengths of the lines in the Web Mercator projection, which are replaced by the lengths along the great circles.
+
+The network has the following properties:
+
+- Nodes: in `endpoints`, `latitude` and `longitude`, the position of the end point; in `substations`, `name`, the name of the substation (empty if not available).
+- Edges: `hifld_id`, the id of the line in HIFLD; `type`, the type of line (e.g. `AC; OVERHEAD`, `AC; UNDERGROUND`, `DC; OVERHEAD`); `status` (`IN SERVICE`, `INACTIVE`, `UNDER CONSTRUCTION`); `owner`, the owner of the line; `voltage`, its voltage (kV; NaN if not available); `volt_class`, the voltage class (e.g. `100-161`, `345`, `DC`); `inferred`, whether the voltage was inferred (`Y` or `N`); `sub_1` and `sub_2`, the names of the substations at the ends of the line, as given in the data; `source`, the sources of the data; `source_date`, the date of the source (Unix time; -1 where missing); `val_method`, the method of validation (e.g. `IMAGERY`); `val_date`, the date of validation (Unix time; -1 where missing); `length`, the length of the line along its course (km, computed from its geometry). Values that are not available are empty strings."""
+tags = ['Technological', 'Power grid', 'Spatial', 'Weighted', 'Multigraph', 'Metadata']
+url = ['https://www.arcgis.com/home/item.html?id=d4090758322c4d32a4cd002ffaa0aa12']
+citation = [('Homeland Infrastructure Foundation-Level Data (HIFLD), "U.S. Electric Power Transmission Lines." US Department of Homeland Security, dataset, archived on ArcGIS Online (2024)', 'https://www.arcgis.com/home/item.html?id=d4090758322c4d32a4cd002ffaa0aa12')]
+# Data of the US government (HIFLD Open was in the public domain); the
+# ArcGIS Online item of the archived copy names the Esri Master License
+# Agreement as its terms of use
+upstream_license = ("US Government work (public domain); Esri Master License Agreement (ArcGIS Online copy)",
+ "https://www.esri.com/en-us/legal/terms/full-master-agreement")
+icon_hash = None
+
+# The layer of the Homeland Infrastructure Foundation-Level Data (HIFLD)
+# Open portal, which was shut down in 2025, as archived by Esri's Federal
+# User Community on ArcGIS Online (last data update 2024-09-30, no further
+# updates). It is downloaded with paged queries (2,000 features each) and
+# stored as one GeoJSON feature per line.
+upstream_prefix = ('https://services2.arcgis.com/FiaPA4ga0iQKduv3/arcgis/rest/'
+ 'services/US_Electric_Power_Transmission_Lines/FeatureServer/0')
+files = [("transmission_lines.geojsonl", "endpoints", None),
+ ("transmission_lines.geojsonl", "substations", None)]
+PAGE = 2000
+
+def _query(**params):
+ params = dict(dict(where="1=1", f="json"), **params)
+ return f"{upstream_prefix}/query?{urllib.parse.urlencode(params)}"
+
+def _page_query(offset):
+ return _query(outFields="*", outSR=4326, orderByFields="OBJECTID_1",
+ resultOffset=offset, resultRecordCount=PAGE, f="geojson")
+
+def fetch_upstream(force=False):
+ name = __name__.split(".")[-1]
+ fname = files[0][0]
+ if process_entry._check_health:
+ with open_url(_page_query(0).replace(f"resultRecordCount={PAGE}",
+ "resultRecordCount=1")) as r:
+ json.load(r)["features"][0]
+ return
+ if not force and upstream_file_exists(name, fname):
+ return
+ with open_url(_query(returnCountOnly="true")) as r:
+ count = json.load(r)["count"]
+ seen = set()
+ # the stored file is written atomically, so that an interrupted download
+ # leaves nothing behind
+ with open_upstream_file(name, fname, "wb") as out:
+ for offset in range(0, count, PAGE):
+ data = retry_download(lambda: open_url(_page_query(offset)).read())
+ feats = json.loads(data)["features"]
+ for feat in feats:
+ oid = feat["properties"]["OBJECTID_1"]
+ if oid in seen:
+ raise RuntimeError(f"feature {oid} returned twice")
+ seen.add(oid)
+ out.write(json.dumps(feat, separators=(",", ":")).encode() + b"\n")
+ if len(seen) != count:
+ raise RuntimeError(f"{len(seen)} features downloaded, {count} expected")
+
+def _ends(geom):
+ "First and last point of a (multi)line geometry."
+ if geom["type"] == "LineString":
+ return geom["coordinates"][0], geom["coordinates"][-1]
+ return geom["coordinates"][0][0], geom["coordinates"][-1][-1]
+
+def _length(geom):
+ "Length of a (multi)line geometry along the great circles (km)."
+ parts = ([geom["coordinates"]] if geom["type"] == "LineString"
+ else geom["coordinates"])
+ total = 0
+ for c in parts:
+ c = numpy.radians(numpy.array(c)[:, :2])
+ lon, lat = c[:, 0], c[:, 1]
+ a = (numpy.sin(numpy.diff(lat) / 2) ** 2 +
+ numpy.cos(lat[:-1]) * numpy.cos(lat[1:]) *
+ numpy.sin(numpy.diff(lon) / 2) ** 2)
+ total += (2 * 6371.0088 * numpy.arcsin(numpy.sqrt(a))).sum()
+ return total
+
+def _str(x):
+ return "" if x is None or x == "NOT AVAILABLE" else x
+
+def _date(x):
+ # dates are given in milliseconds since the epoch
+ return -1 if x is None else x // 1000
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+ name = __name__.split(".")[-1]
+ for fname, alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ nodes = {}
+ coords = []
+ edges = []
+ eprops = defaultdict(list)
+ with open_upstream_file(name, fname, "rb") as f:
+ for line in f:
+ feat = json.loads(line)
+ p = feat["properties"]
+ geom = feat["geometry"]
+ e = []
+ if alt == "endpoints":
+ for x in _ends(geom):
+ # end points are matched after rounding to 1e-7
+ # degrees (about 1 cm), to absorb the noise of the
+ # reprojection
+ k = (round(x[0], 7), round(x[1], 7))
+ if k not in nodes:
+ nodes[k] = len(nodes)
+ coords.append(k)
+ e.append(nodes[k])
+ else:
+ for x in (p["SUB_1"], p["SUB_2"]):
+ # an end without a name gets a node of its own
+ k = _str(x) or (None, len(nodes))
+ if k not in nodes:
+ nodes[k] = len(nodes)
+ e.append(nodes[k])
+ edges.append(e)
+ eprops["hifld_id"].append(p["ID"])
+ for k in ["type", "status", "owner", "volt_class", "inferred",
+ "source", "val_method", "sub_1", "sub_2"]:
+ eprops[k].append(_str(p[k.upper()]))
+ v = p["VOLTAGE"]
+ eprops["voltage"].append(numpy.nan if v is None or v < 0 else v)
+ eprops["source_date"].append(_date(p["SOURCEDATE"]))
+ eprops["val_date"].append(_date(p["VAL_DATE"]))
+ eprops["length"].append(_length(geom))
+ g = Graph(directed=False)
+ g.add_vertex(len(nodes))
+ g.add_edge_list(edges)
+ if alt == "endpoints":
+ coords = numpy.array(coords)
+ g.vp.longitude = g.new_vp("double", vals=coords[:, 0])
+ g.vp.latitude = g.new_vp("double", vals=coords[:, 1])
+ else:
+ g.vp.name = g.new_vp("string", vals=[k if isinstance(k, str) else ""
+ for k in nodes])
+ for k, vals in eprops.items():
+ if k in ("voltage", "length"):
+ t = "double"
+ elif k in ("source_date", "val_date"):
+ t = "int64_t"
+ else:
+ t = "string"
+ g.ep[k] = g.new_ep(t, vals=vals)
+ yield alt, g
+
+def get_pos(g, sbm_fit):
+ # Spatial network: laid out with the default parameters of cfd_layout(),
+ # without the partition of the SBM fit
+ return cfd_layout(g)