History of caida_as_relationships
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] Add the date when each entry was added, and remove icon_hash beff7c9
The dates are those of the commits that first added the entries' files (2020-08-11 for the entries of the initial commit); renamed entries keep the date of their first name. Changed: none
diff --git a/entries/caida_as_relationships/__init__.py b/entries/caida_as_relationships/__init__.py
index ccfa304..ec5bef5 100644
--- a/entries/caida_as_relationships/__init__.py
+++ b/entries/caida_as_relationships/__init__.py
@@ -155,7 +155,7 @@ url = ['https://catalog.caida.org/dataset/as_relationships_serial_1',
citation = [('M. Luckie, B. Huffaker, A. Dhamdhere, V. Giotsas, k. claffy, "AS relationships, customer cones, and validation." Proceedings of the 2013 Internet Measurement Conference (IMC 2013), 243-256 (2013)', 'https://doi.org/10.1145/2504730.2504735'),
('V. Giotsas, S. Zhou, M. Luckie, k. claffy, "Inferring multilateral peering." Proceedings of the 9th ACM Conference on Emerging Networking Experiments and Technologies (CoNEXT 2013), 247-258 (2013)', 'https://doi.org/10.1145/2535372.2535390'),
('CAIDA, "The CAIDA UCSD AS Relationships Dataset (serial-1 and serial-2)." (1998-2026)', 'https://catalog.caida.org/dataset/as_relationships_serial_1')]
-icon_hash = None
+date_added = '2026-10-27'
upstream_license = ("CAIDA Acceptable Use Agreement for publicly accessible datasets",
"https://www.caida.org/about/legal/aua/public_aua/")
upstream_prefix = 'https://publicdata.caida.org/datasets/as-relationships'
[entries] Prefix the properties in the descriptions with (Node), (Edge) or (Graph) bc3dbbd
The lists of properties at the end of the descriptions marked edge properties with "Edge" but left node properties unmarked, or grouped them in "Nodes: `a`, ...; `b`, ..." items, so the kind of a property was often unclear. Every item now starts with (Node), (Edge) or (Graph), followed by the property names, a colon and the explanation, one item per property or group of properties; qualifications (kinds of nodes, networks) follow the names in parentheses. Properties that were only named got a short explanation, and a few items that named the wrong kind of property were corrected.
diff --git a/entries/caida_as_relationships/__init__.py b/entries/caida_as_relationships/__init__.py
index 3b227e4..ccfa304 100644
--- a/entries/caida_as_relationships/__init__.py
+++ b/entries/caida_as_relationships/__init__.py
@@ -143,8 +143,12 @@ The snapshots are named by their date (YYYYMMDD, the first day of the month). No
The networks have the following properties:
-- Nodes: `asn`, the AS number; `clique`, whether the AS belongs to the clique of the largest providers (inferred by the method, or given to it); `ixp`, whether the AS is an Internet exchange point (these are excluded from the inference).
-- Edges: `relationship`, $-1$ for customer–provider (the source node is the customer) and $0$ for peer–peer; `step`, the step of the inference algorithm that inferred the relationship (given in the `serial1_<date>` snapshots of April 2012, of December 2018 to June 2020 and since December 2022, and $0$ in the others, including all the `serial1_v6_<date>` and `serial2_<date>` snapshots): $1$ (peering in the clique), $2$ (initial provider assignment), $3$ (providers for stub ASes), $4$ (provider to larger customer), $5$ (provider-less networks), $6$ (customer–provider for stub–clique relationships), $7$ (folding of peering links), $8$ (everything else is peering); `source` (`serial2_<date>` only), the source of the inference: `bgp`, `mlp` (multilateral peering), `ark` (traceroutes), or several of these separated by commas."""
+- (Node) `asn`: the AS number.
+- (Node) `clique`: whether the AS belongs to the clique of the largest providers (inferred by the method, or given to it).
+- (Node) `ixp`: whether the AS is an Internet exchange point (these are excluded from the inference).
+- (Edge) `relationship`: $-1$ for customer–provider (the source node is the customer) and $0$ for peer–peer.
+- (Edge) `step`: the step of the inference algorithm that inferred the relationship (given in the `serial1_<date>` snapshots of April 2012, of December 2018 to June 2020 and since December 2022, and $0$ in the others, including all the `serial1_v6_<date>` and `serial2_<date>` snapshots): $1$ (peering in the clique), $2$ (initial provider assignment), $3$ (providers for stub ASes), $4$ (provider to larger customer), $5$ (provider-less networks), $6$ (customer–provider for stub–clique relationships), $7$ (folding of peering links), $8$ (everything else is peering).
+- (Edge) `source` (`serial2_<date>` only): the source of the inference: `bgp`, `mlp` (multilateral peering), `ark` (traceroutes), or several of these separated by commas."""
tags = ['Technological', 'Communication', 'Unweighted', 'Temporal', 'Metadata']
url = ['https://catalog.caida.org/dataset/as_relationships_serial_1',
'https://catalog.caida.org/dataset/as_relationships_serial_2']
[entries] Remove unneeded code from the entries 2c69f19
Unneeded "global files" statements, unused imports and variables, f-strings without placeholders, duplicate keys in foodweb_little_rock (keeping the order and the values used), and the second definition of get_lines() in twitter_higgs. Nothing changes in the networks or descriptions. Changed: none
diff --git a/entries/caida_as_relationships/__init__.py b/entries/caida_as_relationships/__init__.py
index 5852cc9..3b227e4 100644
--- a/entries/caida_as_relationships/__init__.py
+++ b/entries/caida_as_relationships/__init__.py
@@ -168,7 +168,6 @@ def fetch_upstream(force=False):
@coerce_props(eforce={"step": "int16_t"})
@annotate()
def parse(alts=None):
- global files
name = __name__.split(".")[-1]
for fnames, alt, fmt in files:
if alts is not None and alt not in alts:
[entries] New entry: caida_as_relationships 920af55
diff --git a/entries/caida_as_relationships/__init__.py b/entries/caida_as_relationships/__init__.py
new file mode 100644
index 0000000..5852cc9
--- /dev/null
+++ b/entries/caida_as_relationships/__init__.py
@@ -0,0 +1,221 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+import bz2
+import re
+
+from .. import *
+
+# All the snapshots of the source (monthly, with gaps)
+dates_serial1 = ["19980101", "19980201", "19980301", "19980401", "19980501", "19980601",
+ "19980701", "19980801", "19980901", "19981001", "19981101", "19981201",
+ "19990201", "19990301", "19990401", "19990501", "19990601", "19990701",
+ "19990801", "19990901", "19991001", "20000101", "20000201", "20000301",
+ "20000401", "20000501", "20000601", "20000701", "20000801", "20000901",
+ "20001001", "20001101", "20001201", "20010101", "20010201", "20010301",
+ "20010401", "20010501", "20010601", "20010701", "20010801", "20010901",
+ "20011001", "20011101", "20011201", "20020101", "20020201", "20020301",
+ "20020401", "20020501", "20020601", "20020701", "20020801", "20020901",
+ "20021001", "20021101", "20021201", "20030101", "20030201", "20030301",
+ "20030401", "20030501", "20030601", "20030701", "20030801", "20030901",
+ "20031001", "20031101", "20031201", "20040101", "20040201", "20040301",
+ "20040401", "20040501", "20040601", "20040701", "20040801", "20040901",
+ "20041001", "20041101", "20041201", "20050101", "20050201", "20050301",
+ "20050401", "20050501", "20050601", "20050701", "20050801", "20050901",
+ "20051001", "20051101", "20051201", "20060101", "20060201", "20060301",
+ "20060401", "20060501", "20060601", "20060701", "20060801", "20060901",
+ "20061001", "20061101", "20061201", "20070101", "20070201", "20070301",
+ "20070401", "20070501", "20070601", "20070701", "20070801", "20070901",
+ "20071001", "20071101", "20071201", "20080101", "20080201", "20080301",
+ "20080401", "20080501", "20080601", "20080701", "20080801", "20080901",
+ "20081001", "20081101", "20081201", "20090101", "20090201", "20090301",
+ "20090401", "20090501", "20090601", "20090701", "20090801", "20090901",
+ "20091001", "20091101", "20091201", "20100101", "20100201", "20100301",
+ "20100401", "20100501", "20100601", "20100701", "20100801", "20100901",
+ "20101001", "20101101", "20101201", "20110101", "20110201", "20110301",
+ "20110401", "20110501", "20110601", "20110701", "20110801", "20110901",
+ "20111001", "20111101", "20111201", "20120101", "20120201", "20120301",
+ "20120401", "20120501", "20120601", "20120701", "20120801", "20120901",
+ "20121001", "20121101", "20121201", "20130101", "20130201", "20130301",
+ "20130401", "20130501", "20130601", "20130701", "20130801", "20130901",
+ "20131001", "20131101", "20131201", "20140101", "20140201", "20140301",
+ "20140401", "20140501", "20140601", "20140701", "20140801", "20140901",
+ "20141001", "20141101", "20141201", "20150101", "20150201", "20150301",
+ "20150401", "20150501", "20150601", "20150701", "20150801", "20150901",
+ "20151001", "20151101", "20151201", "20160101", "20160201", "20160301",
+ "20160401", "20160501", "20160601", "20160701", "20160801", "20160901",
+ "20161001", "20161101", "20161201", "20170101", "20170201", "20170301",
+ "20170401", "20170501", "20170601", "20170701", "20170801", "20170901",
+ "20171001", "20171101", "20171201", "20180101", "20180201", "20180301",
+ "20180401", "20180501", "20180601", "20180701", "20180801", "20180901",
+ "20181001", "20181101", "20181201", "20190101", "20190201", "20190301",
+ "20190401", "20190501", "20190601", "20190701", "20190801", "20190901",
+ "20191001", "20191101", "20191201", "20200101", "20200201", "20200301",
+ "20200401", "20200501", "20200601", "20200701", "20200801", "20200901",
+ "20201001", "20201101", "20201201", "20210101", "20210201", "20210301",
+ "20210401", "20210501", "20210601", "20210701", "20210801", "20210901",
+ "20211001", "20211101", "20211201", "20220101", "20220201", "20220301",
+ "20220401", "20220501", "20220601", "20220701", "20220801", "20220901",
+ "20221001", "20221101", "20221201", "20230101", "20230201", "20230301",
+ "20230401", "20230501", "20230601", "20230701", "20230801", "20230901",
+ "20231001", "20231101", "20231201", "20240101", "20240201", "20240301",
+ "20240401", "20240501", "20240601", "20240701", "20240801", "20240901",
+ "20241001", "20241101", "20241201", "20250101", "20250201", "20250301",
+ "20250401", "20250501", "20250601", "20250701", "20250801", "20250901",
+ "20251001", "20251101", "20251201", "20260101", "20260201", "20260301",
+ "20260401", "20260501", "20260601", "20260701", "20260801", "20260901",
+ "20261001"]
+
+dates_serial1_v6 = ["20150101", "20150201", "20150301", "20150401", "20150501", "20150601",
+ "20150701", "20150801", "20150901", "20151001", "20151101", "20151201",
+ "20160101", "20160201", "20160301", "20160401", "20160501", "20160601",
+ "20160701", "20160801", "20160901", "20161001", "20161101", "20161201",
+ "20170101", "20170201", "20170301", "20170401", "20170501", "20170601",
+ "20170701", "20170801", "20170901", "20171001", "20171101", "20171201",
+ "20180101", "20180201", "20180301", "20180401", "20180501", "20180601",
+ "20180701", "20180801", "20180901", "20181001", "20181101", "20181201",
+ "20190101", "20190201", "20190301", "20190401", "20190501", "20190601",
+ "20190701", "20190801", "20190901", "20191001", "20191101", "20191201",
+ "20200101", "20200201", "20200301", "20200401", "20200501", "20200601",
+ "20200701", "20200801", "20200901", "20201001", "20201101", "20201201",
+ "20210101", "20210201", "20210301", "20210401", "20210501", "20210601",
+ "20210701", "20210801", "20210901", "20211001", "20211101", "20211201",
+ "20220101", "20220201", "20220301", "20220401", "20220501", "20220601",
+ "20220701", "20220801", "20220901", "20221001", "20221101", "20221201",
+ "20230101", "20230201", "20230301", "20230401", "20230501", "20230601",
+ "20230701", "20230801", "20230901", "20231001", "20231101", "20231201",
+ "20240101", "20240201", "20240301", "20240401", "20240501", "20240601",
+ "20240701", "20240801", "20240901", "20241001", "20241101", "20241201",
+ "20250101", "20250201", "20250301", "20250401", "20250501", "20250601",
+ "20250701", "20250801", "20250901", "20251001", "20251101", "20251201",
+ "20260101", "20260201", "20260301", "20260401", "20260501", "20260601",
+ "20260701", "20260801", "20260901", "20261001"]
+
+dates_serial2 = ["20151201", "20160101", "20160201", "20160301", "20160401", "20160501",
+ "20160601", "20160701", "20160801", "20160901", "20161001", "20161101",
+ "20161201", "20170101", "20170201", "20170301", "20170401", "20170501",
+ "20170601", "20170701", "20170801", "20170901", "20171001", "20171101",
+ "20171201", "20180101", "20180201", "20180301", "20180401", "20180501",
+ "20180601", "20180701", "20180801", "20180901", "20181001", "20181101",
+ "20181201", "20190101", "20190201", "20190301", "20190401", "20190501",
+ "20190601", "20190701", "20190801", "20190901", "20191001", "20191101",
+ "20191201", "20200101", "20200401", "20200501", "20200601", "20200701",
+ "20200801", "20200901", "20201001", "20201101", "20201201", "20210101",
+ "20210201", "20210301", "20210401", "20210501", "20210601", "20210701",
+ "20210801", "20210901", "20211001", "20211101", "20211201", "20220101",
+ "20220201", "20220301", "20220401", "20220501", "20220601", "20220701",
+ "20220801", "20220901", "20221001", "20221101", "20221201", "20230101",
+ "20230201", "20230301", "20230401", "20230501", "20230601", "20230701",
+ "20230801", "20230901", "20231001", "20231101", "20231201", "20240101",
+ "20240201", "20240301", "20240401", "20240501", "20240601", "20240701",
+ "20240801", "20240901", "20241001", "20241101", "20241201", "20250101",
+ "20250201", "20250301", "20250401", "20250501", "20250601", "20250701",
+ "20250801", "20250901", "20251001", "20251101", "20251201", "20260101",
+ "20260201", "20260301", "20260401", "20260501", "20260601", "20260701",
+ "20260801", "20260901", "20261001"]
+
+title = "CAIDA AS relationships (1998-2026)"
+description = r"""Monthly snapshots, from January 1998 to October 2026, of the network of business relationships between the Autonomous Systems (AS) of the Internet, from the CAIDA AS Relationships Dataset. An autonomous system is a network or group of networks under a single administrative policy (such as an Internet service provider, a university or a company), identified by its AS number. CAIDA infers the relationships between the ASes that are seen connected in the AS paths of public BGP routing tables (from the RouteViews and RIPE RIS collectors, over the first days of each month), with the method of Luckie et al. (IMC 2013) and its later revisions (e.g. the use of the IANA AS number assignments since 2015, and a fixed set of the largest providers, the "clique", since 2016, inferred automatically again at times between 2017 and 2019; earlier snapshots were not inferred again after these changes). Two kinds of relationships are distinguished: customer–provider (the customer pays the provider for transit to the rest of the Internet) and peer–peer (two ASes exchange the traffic of their own customers, usually without payment).
+
+The data set has three series, all of which are included, with every snapshot available in the source (the snapshots of some months are missing, e.g. those of January, November and December 1999 in `serial-1`, and those of February and March 2020 in `serial-2`, withdrawn by CAIDA because of errors):
+
+- `serial1_<date>` (e.g. `serial1_20260101`): the relationships inferred from the BGP data of IPv4 routes (`serial-1`; 343 snapshots, from January 1998).
+- `serial1_v6_<date>`: the relationships inferred from the BGP data of IPv6 routes (the "v6-stable" files of `serial-1`; 142 snapshots, from January 2015).
+- `serial2_<date>`: the `serial-1` relationships (IPv4) combined with relationships inferred from traceroutes of CAIDA's Archipelago (Ark) infrastructure (until August 2018) and from multilateral peering at Internet exchange points, from looking glass servers (missing from January to November 2023) (`serial-2`; 129 snapshots, from December 2015).
+
+The snapshots are named by their date (YYYYMMDD, the first day of the month). Nodes are ASes, and directed edges are relationships: a customer–provider relationship is an edge from the customer to the provider, and a peer–peer relationship is a single edge in an arbitrary direction (from the smaller to the larger AS number).
+
+The networks have the following properties:
+
+- Nodes: `asn`, the AS number; `clique`, whether the AS belongs to the clique of the largest providers (inferred by the method, or given to it); `ixp`, whether the AS is an Internet exchange point (these are excluded from the inference).
+- Edges: `relationship`, $-1$ for customer–provider (the source node is the customer) and $0$ for peer–peer; `step`, the step of the inference algorithm that inferred the relationship (given in the `serial1_<date>` snapshots of April 2012, of December 2018 to June 2020 and since December 2022, and $0$ in the others, including all the `serial1_v6_<date>` and `serial2_<date>` snapshots): $1$ (peering in the clique), $2$ (initial provider assignment), $3$ (providers for stub ASes), $4$ (provider to larger customer), $5$ (provider-less networks), $6$ (customer–provider for stub–clique relationships), $7$ (folding of peering links), $8$ (everything else is peering); `source` (`serial2_<date>` only), the source of the inference: `bgp`, `mlp` (multilateral peering), `ark` (traceroutes), or several of these separated by commas."""
+tags = ['Technological', 'Communication', 'Unweighted', 'Temporal', 'Metadata']
+url = ['https://catalog.caida.org/dataset/as_relationships_serial_1',
+ 'https://catalog.caida.org/dataset/as_relationships_serial_2']
+citation = [('M. Luckie, B. Huffaker, A. Dhamdhere, V. Giotsas, k. claffy, "AS relationships, customer cones, and validation." Proceedings of the 2013 Internet Measurement Conference (IMC 2013), 243-256 (2013)', 'https://doi.org/10.1145/2504730.2504735'),
+ ('V. Giotsas, S. Zhou, M. Luckie, k. claffy, "Inferring multilateral peering." Proceedings of the 9th ACM Conference on Emerging Networking Experiments and Technologies (CoNEXT 2013), 247-258 (2013)', 'https://doi.org/10.1145/2535372.2535390'),
+ ('CAIDA, "The CAIDA UCSD AS Relationships Dataset (serial-1 and serial-2)." (1998-2026)', 'https://catalog.caida.org/dataset/as_relationships_serial_1')]
+icon_hash = None
+upstream_license = ("CAIDA Acceptable Use Agreement for publicly accessible datasets",
+ "https://www.caida.org/about/legal/aua/public_aua/")
+upstream_prefix = 'https://publicdata.caida.org/datasets/as-relationships'
+
+files = ([(f"serial-1/{d}.as-rel.txt.bz2", f"serial1_{d}", None) for d in dates_serial1] +
+ [(f"serial-1/{d}.as-rel.v6-stable.txt.bz2", f"serial1_v6_{d}", None) for d in dates_serial1_v6] +
+ [(f"serial-2/{d}.as-rel2.txt.bz2", f"serial2_{d}", None) for d in dates_serial2])
+
+def fetch_upstream(force=False):
+ return fetch_upstream_files(__name__.split(".")[-1], upstream_prefix, files,
+ force)
+
+@cache_network()
+@coerce_props(eforce={"step": "int16_t"})
+@annotate()
+def parse(alts=None):
+ global files
+ name = __name__.split(".")[-1]
+ for fnames, alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ clique, ixps = set(), set()
+ edges = []
+ step = 0
+ serial2 = alt.startswith("serial2_")
+ with open_upstream_file(name, fnames, "rb") as fb:
+ with bz2.open(fb, "rt") as f:
+ for line in f:
+ if line.startswith("#"):
+ m = re.match(r"# input clique:(.*)", line)
+ if m is None:
+ m = re.match(r"# inferred clique:(.*)", line)
+ if m is not None:
+ clique.update(int(x) for x in m.group(1).split())
+ m = re.match(r"# IXP ASes:(.*)", line)
+ if m is not None:
+ ixps.update(int(x) for x in m.group(1).split())
+ m = re.match(r"# step (\d+):", line)
+ if m is not None:
+ step = int(m.group(1))
+ continue
+ fields = line.strip().split("|")
+ a, b, r = int(fields[0]), int(fields[1]), int(fields[2])
+ e = [step] + ([fields[3]] if serial2 else [])
+ if r == -1:
+ # provider|customer|-1: stored customer -> provider
+ edges.append((b, a, r, *e))
+ elif r == 0:
+ edges.append((min(a, b), max(a, b), r, *e))
+ else:
+ raise ValueError(f"unknown relationship: {line!r}")
+ g = Graph(directed=True)
+ g.ep.relationship = g.new_ep("int")
+ g.ep.step = g.new_ep("int")
+ eprops = [g.ep.relationship, g.ep.step]
+ if serial2:
+ g.ep.source = g.new_ep("string")
+ eprops.append(g.ep.source)
+ g.vp.asn = g.add_edge_list(edges, hashed=True, hash_type="int",
+ eprops=eprops)
+ g = Graph(g, vorder=g.vp.asn)
+ g.vp.clique = g.new_vp("bool")
+ g.vp.ixp = g.new_vp("bool")
+ for v in g.vertices():
+ g.vp.clique[v] = g.vp.asn[v] in clique
+ g.vp.ixp[v] = g.vp.asn[v] in ixps
+ yield alt, g