logo Netzschleuder network catalog, repository, and centrifuge

History of nwb_roads

The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.

[entries] Lay out the spatial networks with hfd_layout() instead of cfd_layout() by Tiago de Paula Peixoto cc3bcb0
Changed: none
diff --git a/entries/nwb_roads/__init__.py b/entries/nwb_roads/__init__.py
index af7fb5d..2d92fbd 100644
--- a/entries/nwb_roads/__init__.py
+++ b/entries/nwb_roads/__init__.py
@@ -219,6 +219,6 @@ def parse(alts=None):
         yield alt, g
 
 def get_pos(g, sbm_fit):
-    # Spatial network: laid out with the default parameters of cfd_layout(),
+    # Spatial network: laid out with the default parameters of hfd_layout(),
     # without the partition of the SBM fit
-    return cfd_layout(g)
+    return hfd_layout(g)
[entries] Add the date when each entry was added, and remove icon_hash by Tiago de Paula Peixoto beff7c9
The dates are those of the commits that first added the entries' files
(2020-08-11 for the entries of the initial commit); renamed entries keep
the date of their first name.

Changed: none
diff --git a/entries/nwb_roads/__init__.py b/entries/nwb_roads/__init__.py
index 6704a18..af7fb5d 100644
--- a/entries/nwb_roads/__init__.py
+++ b/entries/nwb_roads/__init__.py
@@ -56,7 +56,7 @@ url = ['https://www.rijkswaterstaat.nl/zakelijk/open-data/nationaal-wegenbestand
        'https://downloads.rijkswaterstaatdata.nl/nwb-wegen/geogegevens/shapefile/Nederland_totaal/']
 citation = [('Rijkswaterstaat, "Nationaal Wegenbestand (NWB) Wegen: wegvakken." Dataset, snapshots of 1982-2026 (2026)', 'https://downloads.rijkswaterstaatdata.nl/nwb-wegen/geogegevens/shapefile/Nederland_totaal/')]
 upstream_license = ("Creative Commons CC0 1.0 Universal", "https://creativecommons.org/publicdomain/zero/1.0/")
-icon_hash = None
+date_added = '2026-10-31'
 # the 46 snapshots need about 40 GB of downloads
 huge = True
 
[entries] Prefix the properties in the descriptions with (Node), (Edge) or (Graph) by Tiago de Paula Peixoto bc3dbbd
The lists of properties at the end of the descriptions marked edge
properties with "Edge" but left node properties unmarked, or grouped
them in "Nodes: `a`, ...; `b`, ..." items, so the kind of a property
was often unclear. Every item now starts with (Node), (Edge) or (Graph),
followed by the property names, a colon and the explanation, one item
per property or group of properties; qualifications (kinds of nodes,
networks) follow the names in parentheses. Properties that were only
named got a short explanation, and a few items that named the wrong
kind of property were corrected.
diff --git a/entries/nwb_roads/__init__.py b/entries/nwb_roads/__init__.py
index 1c9eb55..6704a18 100644
--- a/entries/nwb_roads/__init__.py
+++ b/entries/nwb_roads/__init__.py
@@ -29,8 +29,28 @@ Nodes are the junctions, and edges are the road segments; the networks are undir
 
 The networks have the following properties (attributes that are absent from a snapshot are absent from its network):
 
-- Nodes: `jte_id`, the id of the junction; `x` and `y`, its position in the Dutch national grid (Rijksdriehoeksstelsel, EPSG:28992, meters); `latitude` and `longitude`, converted with the approximate formulas of Schreutelkamp and Strang van Hees (accurate to about 1 m).
-- Edges: `wvk_id`, the id of the segment; `begin_date`, the date on which the segment came into existence in the NWB (Unix time; 1982-01-01 for the oldest ones); `length`, its length along its geometry (m); `manager_type`, the type of road manager (`R` state, `P` province, `G` municipality, `W` water board, `T` other); `manager_code` and `manager`, the code and name of the road manager; `road_number`, the road number (for the roads of the state and provinces); `road_part`, the letter of the part of the road along which the hectometer numbering is unique; `hecto_letter`, the letter of the hectometer markers of secondary carriageways (e.g. ramps); `bst_code`, the kind of carriageway (e.g. `HR` main carriageway, `AFR`/`OPR` exit and entry ramps, `VWG` service road, `NRB` roundabout, `BU` bus lane, `FP` cycle path, `VP` footpath, `RP` bridle path, `VD` ferry; empty for most minor roads); `rpe_code`, the relative position (`L`/`R` left or right in the administrative direction of the road, or `N`, `Z`, `O`, `W` for north, south, east, west); `admin_direction`, whether the direction of the segment (from its first to its second junction) agrees with the administrative direction of the road (`H`) or not (`T`); `driving_direction`, the direction of traffic relative to the segment (`H` forward, `T` backward, `B` both, `O` unknown); `street`, the street name; `place`, the place (woonplaats); `municipality_id` and `municipality`, the municipality; and, in the recent snapshots, `road_type`, `frc` (functional road class, 0 to 7) and `fow` (form of way), and `relative_height`, the relative level of the segment (e.g. on a bridge or in a tunnel); `begin_km` and `end_km`, the kilometer markers at the ends of the segments of numbered roads (NaN otherwise). Missing values are empty strings, -1 or NaN."""
+- (Node) `jte_id`: the id of the junction.
+- (Node) `x` and `y`: its position in the Dutch national grid (Rijksdriehoeksstelsel, EPSG:28992, meters).
+- (Node) `latitude` and `longitude`: converted with the approximate formulas of Schreutelkamp and Strang van Hees (accurate to about 1 m).
+- (Edge) `wvk_id`: the id of the segment.
+- (Edge) `begin_date`: the date on which the segment came into existence in the NWB (Unix time; 1982-01-01 for the oldest ones).
+- (Edge) `length`: its length along its geometry (m).
+- (Edge) `manager_type`: the type of road manager (`R` state, `P` province, `G` municipality, `W` water board, `T` other).
+- (Edge) `manager_code`, `manager`: the code and name of the road manager.
+- (Edge) `road_number`: the road number (for the roads of the state and provinces).
+- (Edge) `road_part`: the letter of the part of the road along which the hectometer numbering is unique.
+- (Edge) `hecto_letter`: the letter of the hectometer markers of secondary carriageways (e.g. ramps).
+- (Edge) `bst_code`: the kind of carriageway (e.g. `HR` main carriageway, `AFR`/`OPR` exit and entry ramps, `VWG` service road, `NRB` roundabout, `BU` bus lane, `FP` cycle path, `VP` footpath, `RP` bridle path, `VD` ferry; empty for most minor roads).
+- (Edge) `rpe_code`: the relative position (`L`/`R` left or right in the administrative direction of the road, or `N`, `Z`, `O`, `W` for north, south, east, west).
+- (Edge) `admin_direction`: whether the direction of the segment (from its first to its second junction) agrees with the administrative direction of the road (`H`) or not (`T`).
+- (Edge) `driving_direction`: the direction of traffic relative to the segment (`H` forward, `T` backward, `B` both, `O` unknown).
+- (Edge) `street`: the street name.
+- (Edge) `place`: the place (woonplaats).
+- (Edge) `municipality_id`, `municipality`: the municipality.
+- (Edge) `road_type`, `frc`, `fow` (recent snapshots): the road type, the functional road class (0 to 7) and the form of way.
+- (Edge) `relative_height` (recent snapshots): the relative level of the segment (e.g. on a bridge or in a tunnel).
+- (Edge) `begin_km`, `end_km`: the kilometer markers at the ends of the segments of numbered roads (NaN otherwise).
+- (Edge) Missing values are empty strings, -1 or NaN."""
 tags = ['Transportation', 'Roads', 'Spatial', 'Weighted', 'Multigraph', 'Metadata']
 url = ['https://www.rijkswaterstaat.nl/zakelijk/open-data/nationaal-wegenbestand',
        'https://downloads.rijkswaterstaatdata.nl/nwb-wegen/geogegevens/shapefile/Nederland_totaal/']
[entries] New entry: nwb_roads by Tiago de Paula Peixoto 66038cf
diff --git a/entries/nwb_roads/__init__.py b/entries/nwb_roads/__init__.py
new file mode 100644
index 0000000..1c9eb55
--- /dev/null
+++ b/entries/nwb_roads/__init__.py
@@ -0,0 +1,204 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from .. import *
+import struct
+from datetime import timezone
+
+title = "Dutch national road network (NWB, 1982-2026)"
+description = """The road network of the Netherlands from the Nationaal Wegenbestand (NWB, National Road Database) of Rijkswaterstaat, which contains all the public roads of the country that have a street name or a road number (managed by the state, provinces, municipalities, water boards and others), including separate cycle paths, footpaths and unpaved roads with a street name, and ferry connections; the carriageways of divided roads (mostly state highways) are separate segments. The NWB is a topological network of road segments (wegvakken), each between two junctions (junctie, `JTE_ID_BEG` and `JTE_ID_END`), and is updated monthly.
+
+There is one network for each of the dated snapshots of January 1 of each year from 1982 to 2026 (e.g. `1982`, `2026`), as published by Rijkswaterstaat (the state of the database on that date, reconstructed from its history for the years before the monthly snapshots started in 2011), and one for the latest snapshot, of October 1, 2026 (`2026_10`); the other monthly snapshots since 2011 are not included. The snapshots of the early years are much less complete than the recent ones (e.g. `1982` has 401,161 segments, in 4,807 components, the largest of them with 70,424 of the 306,232 junctions, whereas `2026_10` has 1,667,080 segments and 1,152,537 junctions, almost all in one component).
+
+Nodes are the junctions, and edges are the road segments; the networks are undirected (the direction of traffic is an edge property), and have parallel edges (e.g. the two carriageways of a road, or a road and a parallel cycle path). The positions of the junctions are the end points of the geometries of their segments (in a few cases, the segments that end at the same junction give slightly different positions; the first one is used). The geometries of the segments themselves are not included, and neither are the house numbers, the distances along the roads, the districts and services of Rijkswaterstaat, the route numbers, the sources of the geometries, the OpenLR and BAG references, and the alternative names.
+
+The networks have the following properties (attributes that are absent from a snapshot are absent from its network):
+
+- Nodes: `jte_id`, the id of the junction; `x` and `y`, its position in the Dutch national grid (Rijksdriehoeksstelsel, EPSG:28992, meters); `latitude` and `longitude`, converted with the approximate formulas of Schreutelkamp and Strang van Hees (accurate to about 1 m).
+- Edges: `wvk_id`, the id of the segment; `begin_date`, the date on which the segment came into existence in the NWB (Unix time; 1982-01-01 for the oldest ones); `length`, its length along its geometry (m); `manager_type`, the type of road manager (`R` state, `P` province, `G` municipality, `W` water board, `T` other); `manager_code` and `manager`, the code and name of the road manager; `road_number`, the road number (for the roads of the state and provinces); `road_part`, the letter of the part of the road along which the hectometer numbering is unique; `hecto_letter`, the letter of the hectometer markers of secondary carriageways (e.g. ramps); `bst_code`, the kind of carriageway (e.g. `HR` main carriageway, `AFR`/`OPR` exit and entry ramps, `VWG` service road, `NRB` roundabout, `BU` bus lane, `FP` cycle path, `VP` footpath, `RP` bridle path, `VD` ferry; empty for most minor roads); `rpe_code`, the relative position (`L`/`R` left or right in the administrative direction of the road, or `N`, `Z`, `O`, `W` for north, south, east, west); `admin_direction`, whether the direction of the segment (from its first to its second junction) agrees with the administrative direction of the road (`H`) or not (`T`); `driving_direction`, the direction of traffic relative to the segment (`H` forward, `T` backward, `B` both, `O` unknown); `street`, the street name; `place`, the place (woonplaats); `municipality_id` and `municipality`, the municipality; and, in the recent snapshots, `road_type`, `frc` (functional road class, 0 to 7) and `fow` (form of way), and `relative_height`, the relative level of the segment (e.g. on a bridge or in a tunnel); `begin_km` and `end_km`, the kilometer markers at the ends of the segments of numbered roads (NaN otherwise). Missing values are empty strings, -1 or NaN."""
+tags = ['Transportation', 'Roads', 'Spatial', 'Weighted', 'Multigraph', 'Metadata']
+url = ['https://www.rijkswaterstaat.nl/zakelijk/open-data/nationaal-wegenbestand',
+       'https://downloads.rijkswaterstaatdata.nl/nwb-wegen/geogegevens/shapefile/Nederland_totaal/']
+citation = [('Rijkswaterstaat, "Nationaal Wegenbestand (NWB) Wegen: wegvakken." Dataset, snapshots of 1982-2026 (2026)', 'https://downloads.rijkswaterstaatdata.nl/nwb-wegen/geogegevens/shapefile/Nederland_totaal/')]
+upstream_license = ("Creative Commons CC0 1.0 Universal", "https://creativecommons.org/publicdomain/zero/1.0/")
+icon_hash = None
+# the 46 snapshots need about 40 GB of downloads
+huge = True
+
+# The dated snapshots of the road segments (wegvakken) of the NWB, as
+# shapefiles: the state on January 1 of each year from 1982 to 2026, and the
+# latest one (October 1, 2026); there are monthly snapshots since 2011, of
+# which only these are used. About 40 GB in all.
+upstream_prefix = 'https://downloads.rijkswaterstaatdata.nl/nwb-wegen/geogegevens/shapefile/Nederland_totaal'
+snapshots = {str(y): f"01-01-{y}" for y in range(1982, 2027)}
+snapshots["2026_10"] = "01-10-2026"
+files = [((f"{d}/Wegvakken/Wegvakken.dbf", f"{d}/Wegvakken/Wegvakken.shp"), alt, None)
+         for alt, d in snapshots.items()]
+
+def fetch_upstream(force=False):
+    return fetch_upstream_files(__name__.split(".")[-1], upstream_prefix, files,
+                                force)
+
+def iter_dbf(f):
+    """Iterate over the records of a dBase (DBF) file, read as a stream, as
+    dicts of strings (stripped)."""
+    head = f.read(32)
+    n, hlen, rlen = struct.unpack("<IHH", head[4:12])
+    rest = f.read(hlen - 32)
+    fields = []
+    pos = 0
+    while rest[pos] != 0x0D:
+        d = rest[pos:pos + 32]
+        fields.append((d[:11].split(b"\0")[0].decode(), d[16]))
+        pos += 32
+    for i in range(n):
+        r = f.read(rlen)
+        if len(r) < rlen:
+            raise ValueError("truncated DBF file")
+        if r[0:1] == b"*":      # deleted record
+            yield None
+            continue
+        p = 1
+        row = {}
+        for fname, l in fields:
+            x = r[p:p + l]
+            try:
+                row[fname] = x.decode("utf8").strip()
+            except UnicodeDecodeError:
+                row[fname] = x.decode("cp1252", errors="replace").strip()
+            p += l
+        yield row
+
+def iter_shp_lines(f):
+    """Iterate over the records of a shapefile of (multi)lines, read as a
+    stream, yielding the first and last points and the length of each."""
+    f.read(100)
+    while True:
+        h = f.read(8)
+        if len(h) < 8:
+            return
+        n, length = struct.unpack(">ii", h)
+        c = f.read(2 * length)
+        stype = struct.unpack("<i", c[:4])[0]
+        if stype == 0:          # null shape
+            yield None
+            continue
+        if stype not in (3, 13, 23):        # PolyLine, PolyLineZ, PolyLineM
+            raise ValueError(f"unexpected shape type: {stype}")
+        nparts, npoints = struct.unpack("<ii", c[36:44])
+        parts = list(struct.unpack(f"<{nparts}i", c[44:44 + 4 * nparts])) + [npoints]
+        pts = numpy.frombuffer(c, dtype="<f8", count=2 * npoints,
+                               offset=44 + 4 * nparts).reshape(-1, 2)
+        length = sum(numpy.sqrt((numpy.diff(pts[a:b], axis=0) ** 2).sum(axis=1)).sum()
+                     for a, b in zip(parts[:-1], parts[1:]))
+        yield pts[0], pts[-1], length
+
+# Coefficients of the approximate transformation from the Dutch national
+# grid (Rijksdriehoeksstelsel, EPSG:28992) to WGS84 latitude and longitude
+# (F. H. Schreutelkamp and G. L. Strang van Hees, accurate to about 1 m)
+_K = [(0, 1, 3235.65389), (2, 0, -32.58297), (0, 2, -0.24750),
+      (2, 1, -0.84978), (0, 3, -0.06550), (2, 2, -0.01709), (1, 0, -0.00738),
+      (4, 0, 0.00530), (2, 3, -0.00039), (4, 1, 0.00033), (1, 1, -0.00012)]
+_L = [(1, 0, 5260.52916), (1, 1, 105.94684), (1, 2, 2.45656),
+      (3, 0, -0.81885), (1, 3, 0.05594), (3, 1, -0.05607), (0, 1, 0.01199),
+      (3, 2, -0.00256), (1, 4, 0.00128), (0, 2, 0.00022), (2, 0, -0.00022),
+      (5, 0, 0.00026)]
+
+def rd_to_wgs84(x, y):
+    dx = (numpy.asarray(x) - 155000) * 1e-5
+    dy = (numpy.asarray(y) - 463000) * 1e-5
+    lat = 52.15517440 + sum(c * dx ** p * dy ** q for p, q, c in _K) / 3600
+    lon = 5.38720621 + sum(c * dx ** p * dy ** q for p, q, c in _L) / 3600
+    return lat, lon
+
+# The attributes kept, with their names here
+FIELDS = {"WVK_ID": "wvk_id", "WEGBEHSRT": "manager_type",
+          "WEGBEHCODE": "manager_code", "WEGBEHNAAM": "manager",
+          "WEGNUMMER": "road_number", "WEGDEELLTR": "road_part",
+          "HECTO_LTTR": "hecto_letter", "BST_CODE": "bst_code",
+          "RPE_CODE": "rpe_code", "ADMRICHTNG": "admin_direction",
+          "RIJRICHTNG": "driving_direction", "STT_NAAM": "street",
+          "WPSNAAM": "place", "GME_ID": "municipality_id",
+          "GME_NAAM": "municipality", "WEGTYPE": "road_type",
+          "FRC": "frc", "FOW": "fow", "REL_HOOGTE": "relative_height",
+          "BEGINKM": "begin_km", "EINDKM": "end_km"}
+INT_FIELDS = {"wvk_id", "municipality_id"}
+FLOAT_FIELDS = {"begin_km", "end_km"}
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+    name = __name__.split(".")[-1]
+    for fnames, alt, fmt in files:
+        if alts is not None and alt not in alts:
+            continue
+        nodes = {}
+        pos = []
+        edges = []
+        lengths = []
+        eprops = defaultdict(list)
+        begin = []
+        with open_upstream_file(name, fnames[0], "rb") as fd, \
+             open_upstream_file(name, fnames[1], "rb") as fs:
+            for row, shape in zip(iter_dbf(fd), iter_shp_lines(fs)):
+                if row is None:
+                    continue
+                e = []
+                for k, p in [("JTE_ID_BEG", 0), ("JTE_ID_END", 1)]:
+                    j = int(row[k])
+                    if j not in nodes:
+                        nodes[j] = len(nodes)
+                        # (copied, so as not to keep the record's buffer)
+                        pos.append((float(shape[p][0]), float(shape[p][1]))
+                                   if shape is not None else (numpy.nan, numpy.nan))
+                    e.append(nodes[j])
+                edges.append(e)
+                lengths.append(float(shape[2]) if shape is not None else numpy.nan)
+                d = row["WVK_BEGDAT"]
+                begin.append(int(datetime.strptime(d, "%Y%m%d").replace(tzinfo=timezone.utc).timestamp())
+                             if d else -1)
+                for k, v in FIELDS.items():
+                    if k in row:
+                        eprops[v].append(row[k])
+        g = Graph(directed=False)
+        g.add_vertex(len(nodes))
+        g.add_edge_list(edges)
+        del edges
+        pos = numpy.array(pos, dtype="float")
+        g.vp.jte_id = g.new_vp("int64_t", vals=list(nodes))
+        g.vp.x = g.new_vp("double", vals=pos[:, 0])
+        g.vp.y = g.new_vp("double", vals=pos[:, 1])
+        lat, lon = rd_to_wgs84(pos[:, 0], pos[:, 1])
+        g.vp.latitude = g.new_vp("double", vals=lat)
+        g.vp.longitude = g.new_vp("double", vals=lon)
+        for k, vals in eprops.items():
+            if k in INT_FIELDS:
+                g.ep[k] = g.new_ep("int64_t", vals=[int(x) if x else -1 for x in vals])
+            elif k in FLOAT_FIELDS:
+                g.ep[k] = g.new_ep("double", vals=[float(x) if x else numpy.nan for x in vals])
+            else:
+                g.ep[k] = g.new_ep("string", vals=vals)
+        g.ep.begin_date = g.new_ep("int64_t", vals=begin)
+        g.ep.length = g.new_ep("double", vals=lengths)
+        yield alt, g
+
+def get_pos(g, sbm_fit):
+    # Spatial network: laid out with the default parameters of cfd_layout(),
+    # without the partition of the SBM fit
+    return cfd_layout(g)