History of us_air_traffic

The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.

2026-10-25 [pipeline] Fix KONECT entity files, seekable upstream files and archive formats by Tiago de Paula Peixoto 2f6e19f

- konect.py: each ent.<net>.<entity>.<attr> file becomes a vertex
  property on the side of the entity it describes (per-vertex or per-edge
  files), instead of all of them being written from vertex 0 into `meta`
  (which put e.g. the item names of bibsonomy and reuters on the wrong
  nodes); only leading "%" lines are comments.
- open_upstream_file(..., seekable=True) returns a seekable file (via a
  temporary file) for readers that need one (openpyxl, scipy.io.loadmat,
  zipfile), replacing the per-entry BytesIO copies.
- .xz files can be read again; .xz, .rar and .7z downloads are stored as
  they are, and "x.tar.xz:", "x.rar:" and "x.7z:" member access works
  (with py7zr or the 7z command); caches in the previous layouts are still
  read.
- parsers.py: vprops_from_txt matches rows by name for graphs keyed by
  name; the Wikimedia dump readers (stub histories, SQL tables) are shared
  helpers; new read_doc (tables of Word documents via LibreOffice) and
  read_epanet (EPANET water network files) readers.
- The entries' workarounds are removed (all parse identically).

diff --git a/entries/us_air_traffic/__init__.py b/entries/us_air_traffic/__init__.py
index ca116b6..21bc827 100644
--- a/entries/us_air_traffic/__init__.py
+++ b/entries/us_air_traffic/__init__.py
@@ -21,15 +21,27 @@ from .. import *
 import csv
 
 title = "U.S. air traffic"
-description = """Yearly snapshots of flights among all commercial airports in the United States from 1990 to today. Metadata include passengers, distance, carrier, airport located city, state, and month of the flight.[^icon]
-[^icon]: Description obtained from the [ICON](https://icon.colorado.edu) project."""
+description = r"""Network of domestic air traffic in the United States from January 1990 to early 2020, from the T-100 Domestic Market table of the Bureau of Transportation Statistics (BTS), which records the monthly traffic reported by U.S. air carriers. Nodes are airports and a directed edge $i \to j$ is a record of the table: the traffic carried by one carrier from origin airport $i$ to destination airport $j$ in a given month, for a given service class. Each pair of airports is therefore connected by many parallel edges, one per carrier, month and service class (about 6.4 million edges in total).
+
+The network has the following properties:
+
+- `airport_code`: the three-letter code of the airport.
+- `airport_id`, `airport_seq_id`, `city_market_id`: the BTS identifiers of the airport, of the airport at a given point in time, and of the city market served by the airport.
+- `city_name`, `state_abr`, `state_fips`, `state_nm`, `wac`: the city, the state abbreviation, FIPS code and name, and the BTS world area code of the airport.
+- `passengers`, `freight`, `mail`: the number of passengers, and the amount of freight and mail (in pounds) transported.
+- `distance`: the distance between the airports, in miles, and `distance_group`, the corresponding 500-mile distance interval.
+- `unique_carrier`, `airline_id`, `unique_carrier_name`, `unique_carrier_entity`, `carrier`, `carrier_name`: codes and names identifying the carrier.
+- `region`: the carrier's operating region (e.g. `D` for domestic).
+- `carrier_group`, `carrier_group_new`: the carrier group codes (based on the carrier's revenue).
+- `year`, `quarter`, `month`: the period of the record.
+- `class`: the service class: `F` (scheduled passenger/cargo service), `G` (scheduled all-cargo service), `L` (non-scheduled civilian passenger/cargo service) or `P` (non-scheduled all-cargo service)."""
 tags = ['Transportation', 'Airport', 'Unweighted', 'Metadata', 'Temporal']
 url = 'https://www.transtats.bts.gov/DL_SelectFields.asp?Table_ID=310'
 citation = [('Bureau of Transportation Statistics. "T-100 Domestic Market". https://www.transtats.bts.gov/DL_SelectFields.asp?Table_ID=310 (2017)', 'https://www.transtats.bts.gov/DL_SelectFields.asp?Table_ID=310')]
 icon_hash = '5970fc87549f067e0263fcee'
-ustream_license = None
+upstream_license = None
 upstream_prefix = 'https://www.transtats.bts.gov/DownLoad_Table.asp?Table_ID=310&Has_Group=3&Is_Zipped=0'
-files = [('963574170_T_T100D_MARKET_ALL_CARRIER.zip:963614518_T_T100D_MARKET_ALL_CARRIER.csv', None, None)]
+files = [('963574170_T_T100D_MARKET_ALL_CARRIER.zip:525399827_T_T100D_MARKET_ALL_CARRIER.csv', None, None)]
 
 def fetch_upstream(force=False):
     for fname, alt, fmt in files:
@@ -51,7 +63,7 @@ def fetch_upstream(force=False):
                                      headers={"User-Agent": "Mozilla/5.0"})
         try:
             with urllib.request.urlopen(req, data=data) as f:
-                with open_upstream_file(name, fname, "wb", autocompress=False) as output:
+                with open_upstream_file(name, fname, "wb") as output:
                     shutil.copyfileobj(f, output)
         except urllib.error.HTTPError:
             print("error fetching:", f"{upstream_prefix}/{fname}")
@@ -111,7 +123,7 @@ def parse(alts=None):
             vertices = {g.vp.AIRPORT_CODE[v] : v for v in g.vertices()}
 
             for p in vprops:
-                if "ORIGIN" not in p:
+                if "ORIGIN_" not in p:
                     continue
                 p = p.replace("ORIGIN_", "")
                 g.vp[p] = g.new_vp("string")

2026-10-24 Update copyright information by Tiago de Paula Peixoto 73a4f0d

diff --git a/entries/us_air_traffic/__init__.py b/entries/us_air_traffic/__init__.py
index 346485e..ca116b6 100644
--- a/entries/us_air_traffic/__init__.py
+++ b/entries/us_air_traffic/__init__.py
@@ -1,7 +1,7 @@
 #! /usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2024 Tiago de Paula Peixoto <tiago@skewed.de>
+# Copyright (C) 2020-2026 Tiago de Paula Peixoto <tiago@skewed.de>
 #
 # This program is free software: you can redistribute it and/or modify it under
 # the terms of the GNU Affero General Public License as published by the Free

2024-05-27 Update copyright information by Tiago de Paula Peixoto 3d288ea

diff --git a/entries/us_air_traffic/__init__.py b/entries/us_air_traffic/__init__.py
index 872067e..346485e 100644
--- a/entries/us_air_traffic/__init__.py
+++ b/entries/us_air_traffic/__init__.py
@@ -1,7 +1,7 @@
 #! /usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2023 Tiago de Paula Peixoto <tiago@skewed.de>
+# Copyright (C) 2020-2024 Tiago de Paula Peixoto <tiago@skewed.de>
 #
 # This program is free software: you can redistribute it and/or modify it under
 # the terms of the GNU Affero General Public License as published by the Free

2023-02-19 Update copyright information by Tiago de Paula Peixoto 8d005ae

diff --git a/entries/us_air_traffic/__init__.py b/entries/us_air_traffic/__init__.py
index 6db55ed..872067e 100644
--- a/entries/us_air_traffic/__init__.py
+++ b/entries/us_air_traffic/__init__.py
@@ -1,7 +1,7 @@
 #! /usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2021 Tiago de Paula Peixoto <tiago@skewed.de>
+# Copyright (C) 2020-2023 Tiago de Paula Peixoto <tiago@skewed.de>
 #
 # This program is free software: you can redistribute it and/or modify it under
 # the terms of the GNU Affero General Public License as published by the Free

2021-06-22 Update copyright info by Tiago de Paula Peixoto db2efcc

diff --git a/entries/us_air_traffic/__init__.py b/entries/us_air_traffic/__init__.py
index e501976..6db55ed 100644
--- a/entries/us_air_traffic/__init__.py
+++ b/entries/us_air_traffic/__init__.py
@@ -1,7 +1,7 @@
 #! /usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020 Tiago de Paula Peixoto <tiago@skewed.de>
+# Copyright (C) 2021 Tiago de Paula Peixoto <tiago@skewed.de>
 #
 # This program is free software: you can redistribute it and/or modify it under
 # the terms of the GNU Affero General Public License as published by the Free

2020-11-28 Update citations to use DOIs by Tiago de Paula Peixoto e878874

diff --git a/entries/us_air_traffic/__init__.py b/entries/us_air_traffic/__init__.py
index 24f1480..e501976 100644
--- a/entries/us_air_traffic/__init__.py
+++ b/entries/us_air_traffic/__init__.py
@@ -25,7 +25,7 @@ description = """Yearly snapshots of flights among all commercial airports in th
 [^icon]: Description obtained from the [ICON](https://icon.colorado.edu) project."""
 tags = ['Transportation', 'Airport', 'Unweighted', 'Metadata', 'Temporal']
 url = 'https://www.transtats.bts.gov/DL_SelectFields.asp?Table_ID=310'
-citation = [('Bureau of Transportation Statistics. “T-100 Domestic Market”. https://www.transtats.bts.gov/DL_SelectFields.asp?Table_ID=310 (2017)', 'https://www.transtats.bts.gov/DL_SelectFields.asp?Table_ID=310')]
+citation = [('Bureau of Transportation Statistics. "T-100 Domestic Market". https://www.transtats.bts.gov/DL_SelectFields.asp?Table_ID=310 (2017)', 'https://www.transtats.bts.gov/DL_SelectFields.asp?Table_ID=310')]
 icon_hash = '5970fc87549f067e0263fcee'
 ustream_license = None
 upstream_prefix = 'https://www.transtats.bts.gov/DownLoad_Table.asp?Table_ID=310&Has_Group=3&Is_Zipped=0'

2020-09-15 Change license from GPLv3 to AGPLv3 by Tiago de Paula Peixoto d754be5

diff --git a/entries/us_air_traffic/__init__.py b/entries/us_air_traffic/__init__.py
index 255fe87..24f1480 100644
--- a/entries/us_air_traffic/__init__.py
+++ b/entries/us_air_traffic/__init__.py
@@ -3,18 +3,18 @@
 #
 # Copyright (C) 2020 Tiago de Paula Peixoto <tiago@skewed.de>
 #
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
 #
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
 #
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from .. import *

2020-09-13 Add explicit attribution to descriptions obtained from ICON by Tiago de Paula Peixoto 1645314

diff --git a/entries/us_air_traffic/__init__.py b/entries/us_air_traffic/__init__.py
index ed01ace..255fe87 100644
--- a/entries/us_air_traffic/__init__.py
+++ b/entries/us_air_traffic/__init__.py
@@ -21,7 +21,8 @@ from .. import *
 import csv
 
 title = "U.S. air traffic"
-description = "Yearly snapshots of flights among all commercial airports in the United States from 1990 to today. Metadata include passengers, distance, carrier, airport located city, state, and month of the flight."
+description = """Yearly snapshots of flights among all commercial airports in the United States from 1990 to today. Metadata include passengers, distance, carrier, airport located city, state, and month of the flight.[^icon]
+[^icon]: Description obtained from the [ICON](https://icon.colorado.edu) project."""
 tags = ['Transportation', 'Airport', 'Unweighted', 'Metadata', 'Temporal']
 url = 'https://www.transtats.bts.gov/DL_SelectFields.asp?Table_ID=310'
 citation = [('Bureau of Transportation Statistics. “T-100 Domestic Market”. https://www.transtats.bts.gov/DL_SelectFields.asp?Table_ID=310 (2017)', 'https://www.transtats.bts.gov/DL_SelectFields.asp?Table_ID=310')]

2020-08-11 Initial commit by Tiago de Paula Peixoto cc85872

diff --git a/entries/us_air_traffic/__init__.py b/entries/us_air_traffic/__init__.py
new file mode 100644
index 0000000..ed01ace
--- /dev/null
+++ b/entries/us_air_traffic/__init__.py
@@ -0,0 +1,157 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from .. import *
+
+import csv
+
+title = "U.S. air traffic"
+description = "Yearly snapshots of flights among all commercial airports in the United States from 1990 to today. Metadata include passengers, distance, carrier, airport located city, state, and month of the flight."
+tags = ['Transportation', 'Airport', 'Unweighted', 'Metadata', 'Temporal']
+url = 'https://www.transtats.bts.gov/DL_SelectFields.asp?Table_ID=310'
+citation = [('Bureau of Transportation Statistics. “T-100 Domestic Market”. https://www.transtats.bts.gov/DL_SelectFields.asp?Table_ID=310 (2017)', 'https://www.transtats.bts.gov/DL_SelectFields.asp?Table_ID=310')]
+icon_hash = '5970fc87549f067e0263fcee'
+ustream_license = None
+upstream_prefix = 'https://www.transtats.bts.gov/DownLoad_Table.asp?Table_ID=310&Has_Group=3&Is_Zipped=0'
+files = [('963574170_T_T100D_MARKET_ALL_CARRIER.zip:963614518_T_T100D_MARKET_ALL_CARRIER.csv', None, None)]
+
+def fetch_upstream(force=False):
+    for fname, alt, fmt in files:
+        fname = fname.split(':')[0]
+
+        if not force and os.path.exists(f"cache/upstream/{name}/{fname}"):
+            continue
+
+        data = dict(RawDataTable = "T_T100D_MARKET_ALL_CARRIER",
+                    geo = "All",
+                    time = "All Months",
+                    timename = "Month",
+                    sqlstr = "SELECT PASSENGERS,FREIGHT,MAIL,DISTANCE,UNIQUE_CARRIER,AIRLINE_ID,UNIQUE_CARRIER_NAME,UNIQUE_CARRIER_ENTITY,REGION,CARRIER,CARRIER_NAME,CARRIER_GROUP,CARRIER_GROUP_NEW,ORIGIN_AIRPORT_ID,ORIGIN_AIRPORT_SEQ_ID,ORIGIN_CITY_MARKET_ID,ORIGIN,ORIGIN_CITY_NAME,ORIGIN_STATE_ABR,ORIGIN_STATE_FIPS,ORIGIN_STATE_NM,ORIGIN_WAC,DEST_AIRPORT_ID,DEST_AIRPORT_SEQ_ID,DEST_CITY_MARKET_ID,DEST,DEST_CITY_NAME,DEST_STATE_ABR,DEST_STATE_FIPS,DEST_STATE_NM,DEST_WAC,YEAR,QUARTER,MONTH,DISTANCE_GROUP,CLASS FROM T_T100D_MARKET_ALL_CARRIER",
+                    varlist = "PASSENGERS,FREIGHT,MAIL,DISTANCE,UNIQUE_CARRIER,AIRLINE_ID,UNIQUE_CARRIER_NAME,UNIQUE_CARRIER_ENTITY,REGION,CARRIER,CARRIER_NAME,CARRIER_GROUP,CARRIER_GROUP_NEW,ORIGIN_AIRPORT_ID,ORIGIN_AIRPORT_SEQ_ID,ORIGIN_CITY_MARKET_ID,ORIGIN,ORIGIN_CITY_NAME,ORIGIN_STATE_ABR,ORIGIN_STATE_FIPS,ORIGIN_STATE_NM,ORIGIN_WAC,DEST_AIRPORT_ID,DEST_AIRPORT_SEQ_ID,DEST_CITY_MARKET_ID,DEST,DEST_CITY_NAME,DEST_STATE_ABR,DEST_STATE_FIPS,DEST_STATE_NM,DEST_WAC,YEAR,QUARTER,MONTH,DISTANCE_GROUP,CLASS")
+
+        data = urllib.parse.urlencode(data).encode("ascii")
+
+        req = urllib.request.Request(f"{upstream_prefix}",
+                                     headers={"User-Agent": "Mozilla/5.0"})
+        try:
+            with urllib.request.urlopen(req, data=data) as f:
+                with open_upstream_file(name, fname, "wb", autocompress=False) as output:
+                    shutil.copyfileobj(f, output)
+        except urllib.error.HTTPError:
+            print("error fetching:", f"{upstream_prefix}/{fname}")
+            raise
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+    global files
+    name = __name__.split(".")[-1]
+    for fnames, alt, fmt in files:
+        if alts is not None and alt not in alts:
+            continue
+        if isinstance(fnames, str):
+            fnames = [fnames]
+        with ExitStack() as stack:
+            fs = [stack.enter_context(open_upstream_file(name, fn, "r")) for fn in fnames]
+
+            reader = csv.reader(fs[0])
+            for row in reader:
+                props = row
+                break
+
+            g = Graph()
+
+            vprops = ["ORIGIN_AIRPORT_ID", "ORIGIN_AIRPORT_SEQ_ID",
+                      "ORIGIN_CITY_MARKET_ID", "ORIGIN", "ORIGIN_CITY_NAME",
+                      "ORIGIN_STATE_ABR", "ORIGIN_STATE_FIPS",
+                      "ORIGIN_STATE_NM", "ORIGIN_WAC", "DEST_AIRPORT_ID",
+                      "DEST_AIRPORT_SEQ_ID", "DEST_CITY_MARKET_ID", "DEST",
+                      "DEST_CITY_NAME", "DEST_STATE_ABR", "DEST_STATE_FIPS",
+                      "DEST_STATE_NM", "DEST_WAC"]
+
+            for p in props:
+                if p not in vprops:
+                    g.ep[p] = g.new_ep("string")
+
+            def get_edges():
+                reader = csv.reader(fs[0])
+                for row in reader:
+                    vals = []
+                    for x, p in zip(row, props):
+                        if p == "ORIGIN":
+                            s = x
+                            continue
+                        if p == "DEST":
+                            t = x
+                            continue
+                        if p not in vprops:
+                            vals.append(x)
+                    yield [s, t] + vals
+
+            g.vp["AIRPORT_CODE"] = g.add_edge_list(get_edges(), hashed=True,
+                                                   eprops=list(g.ep.values()))
+
+            vertices = {g.vp.AIRPORT_CODE[v] : v for v in g.vertices()}
+
+            for p in vprops:
+                if "ORIGIN" not in p:
+                    continue
+                p = p.replace("ORIGIN_", "")
+                g.vp[p] = g.new_vp("string")
+
+            fs[0].seek(0)
+            reader = csv.reader(fs[0])
+            for row in reader:
+                break
+            for row in reader:
+                svals = {}
+                tvals = {}
+                for x, p in zip(row, props):
+                    if p == "ORIGIN":
+                        s = vertices[x]
+                        continue
+                    if p == "DEST":
+                        t = vertices[x]
+                        continue
+                    if p not in vprops:
+                        continue
+                    if "ORIGIN" in p:
+                        p = p.replace("ORIGIN_", "")
+                        svals[p] = x
+                    else:
+                        p = p.replace("DEST_", "")
+                        tvals[p] = x
+                for p, x in svals.items():
+                    g.vp[p][s] = x
+                for p, x in tvals.items():
+                    g.vp[p][t] = x
+
+            for p in ["DISTANCE", "MAIL", "FREIGHT", "PASSENGERS"]:
+                g.ep[p] = g.ep[p].copy("double")
+            del g.ep[""]
+
+            for p in list(g.vp.keys()):
+                g.vp[p.lower()] = g.vp[p]
+                del g.vp[p]
+
+            for p in list(g.ep.keys()):
+                g.ep[p.lower()] = g.ep[p]
+                del g.ep[p]
+
+            yield alt, g