History of gridkit
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] New entry: gridkit 118a085
diff --git a/entries/gridkit/__init__.py b/entries/gridkit/__init__.py
new file mode 100644
index 0000000..8988e2d
--- /dev/null
+++ b/entries/gridkit/__init__.py
@@ -0,0 +1,179 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from .. import *
+import csv
+import re
+
+title = "GridKit high-voltage grids of Europe and North America (2016)"
+description = """Networks of the high-voltage electricity transmission grids of Europe and North America, extracted from OpenStreetMap (OSM) (`europe` and `north_america`) and from the interactive map of ENTSO-E (`entsoe`, see below) by B. Wiegmans with GridKit, a toolkit developed in the context of the SciGRID project that transforms the OSM power objects (lines, cables and stations) into a topologically connected network model by spatial analysis. The OSM extracts were made in March 2016, and contain the high-voltage part of the grids as selected by GridKit (according to its documentation, lines of 220 kV and above, at any frequency other than the 16.7 Hz of railway power; a part of the lines have no voltage in OSM, and some lines of lower voltage remain).
+
+In the OSM extracts, a node is a station or a junction of lines, and an edge is a line or cable between two nodes. There is one network for each continent, `europe` and `north_america`. The networks are undirected; some pairs of nodes are joined by several lines (parallel edges), and the networks are not connected (they have 86 and 41 components, respectively). The geometries of the lines (their courses as polylines) are not included, and neither are the columns of the line parameters (resistance, reactance, capacitance, thermal limit, and the OSM relation), which are empty in the extracts. Most attributes are taken from the OSM tags, and can hold several values separated by semicolons (e.g. for lines with several circuits of different voltages).
+
+The networks have the following properties:
+
+The network `entsoe` comes from another dataset of B. Wiegmans: an unofficial extract (of May 11, 2016) of the interactive map of the European power system of ENTSO-E (the European Network of Transmission System Operators for Electricity), including to a limited extent North Africa and the Middle East, processed by GridKit to form a topologically connected network. It is neither approved nor endorsed by ENTSO-E, and, according to its documentation, its coordinates follow the map, which favors topological clarity over geographical accuracy, its voltages are the lower bounds of the ranges of the map (not available for all lines, especially DC lines), and its transformers and the connections of the generators are not in the map, but were inferred (transformers between buses of different voltages within the same station, and generators connected to the nearest station at its lowest voltage). Its nodes are buses (a station can have several buses, of different voltages, connected by transformers; junctions of lines are synthetic stations of type `joint`), and its edges are the lines and cables between them and the transformers within stations (edge property `kind`). It has parallel edges and is not connected (139 components). The generators are kept as node properties of the buses they are connected to, and the geometries of the lines are not included.
+
+The networks `europe` and `north_america` have the following properties:
+
+- Nodes: `vid`, the GridKit id of the node; `type`, the kind of node (`joint`, `merge`, `substation`, `sub_station`, `station`, `plant`, ...); `voltage`, the voltage(s) (V); `frequency`, the frequency(-ies) (Hz); `name`, `operator` and `ref`, the name, operator and reference of the station in OSM; `latitude` and `longitude`, the geographical position.
+- Edges: `lid`, the GridKit id of the line; `voltage`, the voltage(s) (V); `cables`, the number(s) of cables; `wires`, the number(s) of wires per cable bundle; `frequency`, the frequency(-ies) (Hz); `name`, `operator` and `ref`, the name, operator and reference of the line in OSM; `length`, the length of the line (m). Missing values are empty strings (or NaN).
+
+The network `entsoe` has the following properties:
+
+- Nodes: `bus_id` and `station_id`, the ids of the bus and of its station; `type`, the type of station (e.g. `Substation`, `joint`, or a kind of power plant); `voltage`, the voltage (kV; empty if not available); `dc`, whether the bus is a HVDC terminal; `under_construction`, whether the station is under construction; `name` and `name_national`, the name of the station in English and in the national language; `country`, the country code; `tso`, the transmission system operator (where given); `tags`, all the attributes of the map (in hstore format); `latitude` and `longitude`, the position; `generator_types` and `generator_capacities`, the types and capacities (MW; -1 or NaN if unknown) of the generators connected to the bus.
+- Edges: `kind`, `line` (a line or cable), `transformer` or `ac/dc` (a converter); `lid`, the id of the line or transformer; `voltage`, the voltage of the line (kV), or the voltages of the two buses of a transformer separated by a semicolon; `circuits`, the number of circuits of the line (0 for transformers); `dc`, whether the line is a HVDC line, or the transformer a converter; `underground`, whether the line is an underground cable; `under_construction`, whether the line is under construction; `length`, the length of the line (m; NaN for transformers); `symbol`, the description of the line in the map (e.g. `380-400 kV Transmission Line, 1 Circuit`); `tie_line`, whether the line is a tie line between countries; `tags`, all the attributes of the line in the map (in hstore format)."""
+tags = ['Technological', 'Power grid', 'Spatial', 'Weighted', 'Multigraph', 'Metadata']
+url = ['https://zenodo.org/records/47317', 'https://zenodo.org/records/55853',
+ 'https://github.com/bdw/GridKit']
+citation = [('B. Wiegmans, "GridKit: European and North-American extracts." Zenodo, dataset (2016)', 'https://doi.org/10.5281/zenodo.47317'),
+ ('B. Wiegmans, "GridKit 1.0 \'for scientists\'." Zenodo, software (2016)', 'https://doi.org/10.5281/zenodo.47263'),
+ ('B. Wiegmans, "GridKit extract of ENTSO-E interactive map." Zenodo, dataset (2016)', 'https://doi.org/10.5281/zenodo.55853')]
+# ODbL for the OpenStreetMap extracts, CC BY 4.0 for the ENTSO-E extract
+upstream_license = ('Open Data Commons Open Database License (ODbL) v1.0 (europe, north_america); Creative Commons Attribution 4.0 International (CC BY 4.0) (entsoe)',
+ 'https://opendatacommons.org/licenses/odbl/1-0/')
+icon_hash = None
+upstream_prefix = {'europe': 'https://zenodo.org/records/47317/files',
+ 'north_america': 'https://zenodo.org/records/47317/files',
+ 'entsoe': 'https://zenodo.org/records/55853/files'}
+# (the European archive is misspelled upstream)
+files = [((f"gridkit_euorpe.zip:gridkit_europe-highvoltage-vertices.csv",
+ f"gridkit_euorpe.zip:gridkit_europe-highvoltage-links.csv"),
+ "europe", None),
+ ((f"gridkit_north_america.zip:gridkit_north_america-highvoltage-vertices.csv",
+ f"gridkit_north_america.zip:gridkit_north_america-highvoltage-links.csv"),
+ "north_america", None),
+ (("entsoe.zip:buses.csv", "entsoe.zip:links.csv",
+ "entsoe.zip:generators.csv", "entsoe.zip:transformers.csv"),
+ "entsoe", None)]
+
+def fetch_upstream(force=False):
+ for f in files:
+ fetch_upstream_files(__name__.split(".")[-1], upstream_prefix[f[1]],
+ [f], force)
+
+class quote_dialect(csv.excel):
+ # the ENTSO-E extract quotes strings with single quotes
+ quotechar = "'"
+
+def read_entsoe(name, fname):
+ with open_upstream_file(name, fname, "r") as f:
+ return list(csv.DictReader(f, dialect=quote_dialect))
+
+def hstore(s):
+ # the "tags" columns: hstore-encoded dictionaries ("key"=>"value", ...)
+ return {k: v.replace('\\"', '"').replace('\\\\', '\\') for k, v in
+ re.findall(r'"((?:[^"\\]|\\.)*)"=>"((?:[^"\\]|\\.)*)"', s)}
+
+def point(wkt):
+ lon, lat = wkt.removeprefix("POINT(").removesuffix(")").split()
+ return float(lat), float(lon)
+
+def parse_entsoe(name, fnames):
+ buses, links, generators, transformers = [read_entsoe(name, f) for f in fnames]
+ idx = {r["bus_id"]: i for i, r in enumerate(buses)}
+ g = Graph(directed=False)
+ g.add_vertex(len(buses))
+ tags = [hstore(r["tags"]) for r in buses]
+ g.vp.bus_id = g.new_vp("int", vals=[int(r["bus_id"]) for r in buses])
+ g.vp.station_id = g.new_vp("int", vals=[int(r["station_id"]) for r in buses])
+ g.vp.type = g.new_vp("string", vals=[r["symbol"] for r in buses])
+ g.vp.voltage = g.new_vp("string", vals=[r["voltage"] for r in buses])
+ g.vp.dc = g.new_vp("bool", vals=[r["dc"] == "t" for r in buses])
+ g.vp.under_construction = g.new_vp("bool", vals=[r["under_construction"] == "t"
+ for r in buses])
+ for p, k in [("name", "name_eng"), ("name_national", "name_nat"),
+ ("country", "country"), ("tso", "tso")]:
+ g.vp[p] = g.new_vp("string", vals=[t.get(k, "").strip() for t in tags])
+ g.vp.tags = g.new_vp("string", vals=[r["tags"] for r in buses])
+ pos = [point(r["geometry"]) for r in buses]
+ g.vp.latitude = g.new_vp("double", vals=[p[0] for p in pos])
+ g.vp.longitude = g.new_vp("double", vals=[p[1] for p in pos])
+ g.vp.generator_types = g.new_vp("vector<string>")
+ g.vp.generator_capacities = g.new_vp("vector<double>")
+ for r in generators:
+ v = g.vertex(idx[r["bus_id"]])
+ g.vp.generator_types[v].append(r["symbol"])
+ g.vp.generator_capacities[v].append(float(r["capacity"] or "nan"))
+ edges = []
+ for r in links:
+ t = hstore(r["tags"])
+ edges.append((idx[r["src_bus_id"]], idx[r["dst_bus_id"]], "line",
+ int(r["link_id"]), r["voltage"], int(r["circuits"]),
+ r["dc"] == "t", r["underground"] == "t",
+ r["under_construction"] == "t", float(r["length_m"]),
+ t.get("symbol", ""), t.get("tie_line", "") == "1",
+ r["tags"]))
+ for r in transformers:
+ edges.append((idx[r["src_bus_id"]], idx[r["dst_bus_id"]], r["symbol"],
+ int(r["transformer_id"]),
+ f'{r["src_voltage"]};{r["dst_voltage"]}', 0,
+ r["src_dc"] == "t" or r["dst_dc"] == "t", False, False,
+ numpy.nan, "", False, ""))
+ g.add_edge_list([e[:2] for e in edges])
+ for i, (p, t) in enumerate([("kind", "string"), ("lid", "int"),
+ ("voltage", "string"), ("circuits", "int"),
+ ("dc", "bool"), ("underground", "bool"),
+ ("under_construction", "bool"),
+ ("length", "double"), ("symbol", "string"),
+ ("tie_line", "bool"), ("tags", "string")]):
+ g.ep[p] = g.new_ep(t, vals=[e[2 + i] for e in edges])
+ return g
+
+def to_float(x):
+ return float(x) if x != "" else numpy.nan
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+ global files
+ name = __name__.split(".")[-1]
+ for fnames, alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ if alt == "entsoe":
+ yield alt, parse_entsoe(name, fnames)
+ continue
+ with open_upstream_file(name, fnames[0], "r") as f:
+ vertices = list(csv.DictReader(f))
+ with open_upstream_file(name, fnames[1], "r") as f:
+ links = list(csv.DictReader(f))
+ idx = {r["v_id"]: i for i, r in enumerate(vertices)}
+ g = Graph(directed=False)
+ g.add_vertex(len(vertices))
+ g.vp.vid = g.new_vp("int", vals=[int(r["v_id"]) for r in vertices])
+ g.vp.type = g.new_vp("string", vals=[r["typ"] for r in vertices])
+ for k in ["voltage", "frequency", "name", "operator", "ref"]:
+ g.vp[k] = g.new_vp("string", vals=[r[k] for r in vertices])
+ g.vp.latitude = g.new_vp("double", vals=[float(r["lat"]) for r in vertices])
+ g.vp.longitude = g.new_vp("double", vals=[float(r["lon"]) for r in vertices])
+ g.add_edge_list([(idx[r["v_id_1"]], idx[r["v_id_2"]]) for r in links])
+ g.ep.lid = g.new_ep("int", vals=[int(r["l_id"]) for r in links])
+ for k in ["voltage", "cables", "wires", "frequency", "name",
+ "operator", "ref"]:
+ g.ep[k] = g.new_ep("string", vals=[r[k] for r in links])
+ g.ep.length = g.new_ep("double", vals=[to_float(r["length_m"])
+ for r in links])
+ yield alt, g
+
+
+def get_pos(g, sbm_fit):
+ # Spatial network: laid out with the default parameters of cfd_layout(),
+ # without the partition of the SBM fit
+ return cfd_layout(g)