logo Netzschleuder network catalog, repository, and centrifuge

History of divvy

The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.

[entries] New entry: divvy by Tiago de Paula Peixoto e69fd8a
diff --git a/entries/divvy/__init__.py b/entries/divvy/__init__.py
new file mode 100644
index 0000000..bcb8b64
--- /dev/null
+++ b/entries/divvy/__init__.py
@@ -0,0 +1,243 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from .. import *
+import re
+from collections import Counter
+
+title = "Divvy bicycle sharing trips (Chicago, 2013-2026)"
+description = """Monthly networks of the trips made with Divvy, the bicycle sharing system of Chicago (operated by Lyft), from the trip data published by the operator: one network for each month from June 2013, when the system opened, to September 2026, named `<year>_<month>` (e.g. `2013_06`), 160 networks in all.
+
+Each node is a station, and each directed edge goes from the station where a bicycle was taken out to the station where it was returned, weighted by the number of such trips in the month (trips that return to the same station are self-loops); trips are assigned to the month in which they started. The networks contain 49,171,160 trips in total. Left out are the 9,035,807 trips (all since 2020) without a start or an end station, mostly of electric bicycles taken out or left away from a station, and the details of individual trips (times, durations, bicycles, and the age and gender of the riders, given until 2019). The trip data until 2019 were published per quarter or half year and are split here by month.
+
+The station ids changed in December 2020 (from the numbers used until then to new codes), so the stations of different months can be matched by `station_id` only within each period. Until March 2020, the coordinates of the stations are those of the station tables published with the trip data (for 2014, the table of the second half of the year; for 2018 and 2019, for which none was published, the table of the second half of 2017, and for the stations missing from it, the coordinates in the trips of early 2020); since then they are the most common coordinates of the trips that start or end at the station (those of electric bicycles are recorded by GPS). The coordinates of a few stations are unknown.
+
+The station-to-station networks are not planar spatial networks (trips may connect any two stations), so they are laid out as other networks, with the coordinates as vertex properties.
+
+The files of this entry are available only with an access token, since the Divvy Data License Agreement forbids redistributing the data as a stand-alone dataset.
+
+The networks have the following properties:
+
+- Nodes: `station_id`, the id of the station; `name`, its name (the most common one in the month); `lat` and `lon`, its geographic coordinates (NaN if unknown).
+- Edges: `weight`, the number of trips; `member_trips`, the number of trips by members, i.e. subscribers with an annual membership (the others are by casual users, with single rides or day passes)."""
+tags = ['Transportation', 'Travel', 'Weighted', 'Metadata']
+url = ['https://divvybikes.com/system-data', 'https://divvy-tripdata.s3.amazonaws.com/index.html']
+citation = [('Lyft Bikes and Scooters, "Divvy System Data." (2026)', 'https://divvybikes.com/system-data')]
+icon_hash = None
+upstream_license = ("Divvy Data License Agreement (no redistribution as a stand-alone dataset)", "https://divvybikes.com/data-license-agreement")
+restricted = True
+
+upstream_prefix = 'https://divvy-tripdata.s3.amazonaws.com'
+
+def months(start, end):
+    y, m = start
+    while (y, m) <= end:
+        yield y, m
+        y, m = (y, m + 1) if m < 12 else (y + 1, 1)
+
+def archive(y, m):
+    "The archive with the trips of month (y, m)."
+    q = (m - 1) // 3 + 1
+    h = "Q1Q2" if m <= 6 else "Q3Q4"
+    if y == 2013:
+        return "Divvy_Stations_Trips_2013.zip"
+    if y == 2014:
+        return f"Divvy_Stations_Trips_2014_{h}.zip"
+    if y == 2015:
+        return "Divvy_Trips_2015-Q1Q2.zip" if m <= 6 else "Divvy_Trips_2015_Q3Q4.zip"
+    if y in (2016, 2017):
+        return f"Divvy_Trips_{y}_{h}.zip"
+    if y in (2018, 2019) or (y, m) <= (2020, 3):
+        return f"Divvy_Trips_{y}_Q{q}.zip"
+    return f"{y}{m:02d}-divvy-tripdata.zip"
+
+def station_table(y, m):
+    """The station table (with the coordinates of the stations) for the
+    trips of month (y, m), until March 2020, when the trip files do not give
+    the coordinates; none was published for 2018 and 2019, for which the last
+    one, of the second half of 2017, is used (see parse())."""
+    if y == 2013:
+        return "Divvy_Stations_Trips_2013.zip:Divvy_Stations_Trips_2013/Divvy_Stations_2013.csv"
+    if y == 2014:
+        # (the table of the first half of 2014 is a spreadsheet; the one of
+        # the second half is used for the whole year)
+        return "Divvy_Stations_Trips_2014_Q3Q4.zip:Divvy_Stations_Trips_2014_Q3Q4/Divvy_Stations_2014-Q3Q4.csv"
+    if y == 2015:
+        return "Divvy_Trips_2015-Q1Q2.zip:Divvy_Stations_2015.csv"
+    if y == 2016 and m <= 6:
+        return "Divvy_Trips_2016_Q1Q2.zip:Divvy_Trips_2016_Q1Q2/Divvy_Stations_2016_Q1Q2.csv"
+    if y == 2016:
+        return f"Divvy_Trips_2016_Q3Q4.zip:Divvy_Stations_2016_Q{(m - 1) // 3 + 1}.csv"
+    if y == 2017 and m <= 6:
+        return "Divvy_Trips_2017_Q1Q2.zip:Divvy_Stations_2017_Q1Q2.csv"
+    return "Divvy_Trips_2017_Q3Q4.zip:Divvy_Stations_2017_Q3Q4.csv"
+
+# The trips of the first quarter of 2020, which give the coordinates of the
+# stations (with the same ids as before), used for the stations of 2018 and
+# 2019 that are missing from the last station table
+coords_2020 = "Divvy_Trips_2020_Q1.zip:Divvy_Trips_2020_Q1.csv"
+
+files = []
+for y, m in months((2013, 6), (2026, 9)):
+    if (y, m) < (2020, 1):
+        fnames = (archive(y, m), station_table(y, m))
+        if y >= 2018:
+            fnames += (coords_2020,)
+    else:
+        fnames = (archive(y, m),)
+    files.append((fnames, f"{y}_{m:02d}", None))
+
+def fetch_upstream(force=False):
+    return fetch_upstream_files(__name__.split(".")[-1], upstream_prefix, files,
+                                force)
+
+def trip_month(t):
+    "The (year, month) of a time given as YYYY-MM-DD ... or M/D/YYYY ..."
+    if t[4:5] == "-":
+        return int(t[:4]), int(t[5:7])
+    m, d, y = t.split(" ", 1)[0].split("/")
+    return int(y), int(m)
+
+def coords_mode(c):
+    "The most common valid coordinates in the counter `c`, or NaN."
+    for (a, b), n in c.most_common():
+        try:
+            lat, lon = float(a), float(b)
+        except ValueError:
+            continue
+        if lat != 0 and lon != 0:
+            return lat, lon
+    return numpy.nan, numpy.nan
+
+def parse_trips(rows, y, m, stations):
+    """Aggregate the trips of month (y, m) in `rows` (dicts with the start and
+    end stations, the start time, the user type and the coordinates, if
+    given) into a directed network of stations, weighted by the number of
+    trips. `stations` maps station ids to coordinates (for the trip files
+    without them)."""
+    counts = defaultdict(lambda: [0, 0])
+    names = defaultdict(Counter)
+    coords = defaultdict(Counter)
+    ntrips = nskipped = 0
+    for r in rows:
+        if trip_month(r["time"]) != (y, m):
+            continue
+        s, t = r["start_id"].strip(), r["end_id"].strip()
+        if s == "" or t == "":
+            nskipped += 1
+            continue
+        c = counts[(s, t)]
+        c[0] += 1
+        if r["user"].strip().lower() in ("subscriber", "member"):
+            c[1] += 1
+        ntrips += 1
+        for v, p in ((s, "start"), (t, "end")):
+            names[v][r[f"{p}_name"].strip()] += 1
+            if f"{p}_lat" in r:
+                coords[v][(r[f"{p}_lat"], r[f"{p}_lon"])] += 1
+    g = Graph(directed=True)
+    g.vp.station_id = g.new_vp("string")
+    g.vp.name = g.new_vp("string")
+    g.vp.lat = g.new_vp("double")
+    g.vp.lon = g.new_vp("double")
+    vs = {}
+    for v in sorted(names):
+        u = vs[v] = int(g.add_vertex())
+        g.vp.station_id[u] = v
+        g.vp.name[u] = names[v].most_common(1)[0][0]
+        lat, lon = coords_mode(coords[v]) if v in coords else stations.get(v, (numpy.nan, numpy.nan))
+        g.vp.lat[u] = lat
+        g.vp.lon[u] = lon
+    g.ep.weight = g.new_ep("int")
+    g.ep.member_trips = g.new_ep("int")
+    g.add_edge_list(((vs[s], vs[t], c[0], c[1])
+                     for (s, t), c in sorted(counts.items())),
+                    eprops=[g.ep.weight, g.ep.member_trips])
+    return g, ntrips, nskipped
+
+# Columns of the trip files until 2019 (by position: their names vary)
+OLD_COLUMNS = {"time": 1, "start_id": 5, "start_name": 6, "end_id": 7,
+               "end_name": 8, "user": 9}
+# Columns of the trip files since 2020
+NEW_COLUMNS = {"time": "started_at", "start_id": "start_station_id",
+               "start_name": "start_station_name", "end_id": "end_station_id",
+               "end_name": "end_station_name", "user": "member_casual",
+               "start_lat": "start_lat", "start_lon": "start_lng",
+               "end_lat": "end_lat", "end_lon": "end_lng"}
+
+def read_rows(z):
+    "The trips in the trip files of the zip archive `z`, as dicts."
+    for member in sorted(z.namelist()):
+        base = member.rsplit("/", 1)[-1]
+        if (member.startswith("__MACOSX") or not base.endswith(".csv") or
+            "Stations" in base):
+            continue
+        with io.TextIOWrapper(z.open(member), "utf-8-sig", errors="replace") as f:
+            reader = csv.reader(f)
+            header = next(reader)
+            if "started_at" in header:
+                idx = {k: header.index(c) for k, c in NEW_COLUMNS.items()}
+            else:
+                idx = OLD_COLUMNS
+            n = max(idx.values()) + 1
+            for row in reader:
+                if len(row) < n:
+                    continue
+                yield {k: row[i] for k, i in idx.items()}
+
+def read_stations(f):
+    "Station id -> coordinates, from a station table."
+    stations = {}
+    for r in csv.DictReader(f):
+        try:
+            stations[r["id"].strip()] = (float(r["latitude"]), float(r["longitude"]))
+        except (ValueError, KeyError, TypeError):
+            pass
+    return stations
+
+@cache_network()
+@coerce_props(vforce={"station_id": "string"})
+@annotate()
+def parse(alts=None):
+    global files
+    name = __name__.split(".")[-1]
+    for fnames, alt, fmt in files:
+        if alts is not None and alt not in alts:
+            continue
+        y, m = int(alt[:4]), int(alt[5:])
+        stations = {}
+        if len(fnames) > 1:
+            with open_upstream_file(name, fnames[1], "r", encoding="utf-8-sig") as f:
+                stations = read_stations(f)
+        if len(fnames) > 2:
+            # coordinates of the stations missing from the table, from the
+            # trips of early 2020
+            c = defaultdict(Counter)
+            with open_upstream_file(name, fnames[2], "r") as f:
+                for r in csv.DictReader(f):
+                    for p in ("start", "end"):
+                        c[r[f"{p}_station_id"].strip()][(r[f"{p}_lat"], r[f"{p}_lng"])] += 1
+            for v, cv in c.items():
+                if v not in stations:
+                    stations[v] = coords_mode(cv)
+        with ExitStack() as stack:
+            f = stack.enter_context(open_upstream_file(name, fnames[0], "rb"))
+            z = stack.enter_context(zipfile.ZipFile(f))
+            g, ntrips, nskipped = parse_trips(read_rows(z), y, m, stations)
+        yield alt, g