History of wikilinkgraphs
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] Add the date when each entry was added, and remove icon_hash beff7c9
The dates are those of the commits that first added the entries' files (2020-08-11 for the entries of the initial commit); renamed entries keep the date of their first name. Changed: none
diff --git a/entries/wikilinkgraphs/__init__.py b/entries/wikilinkgraphs/__init__.py
index 54639e4..e95c55f 100644
--- a/entries/wikilinkgraphs/__init__.py
+++ b/entries/wikilinkgraphs/__init__.py
@@ -38,7 +38,7 @@ tags = ['Informational', 'Web graph', 'Unweighted', 'Metadata']
url = ['https://zenodo.org/records/2539424']
citation = [('C. Consonni, D. Laniado, A. Montresor, "WikiLinkGraphs: A Complete, Longitudinal and Multi-Language Dataset of the Wikipedia Link Networks." Proceedings of the International AAAI Conference on Web and Social Media 13, 598-607 (2019)',
'https://doi.org/10.1609/icwsm.v13i01.3257')]
-icon_hash = None
+date_added = '2026-10-28'
upstream_license = ('CC BY 4.0', 'https://creativecommons.org/licenses/by/4.0/')
upstream_prefix = 'https://zenodo.org/records/2539424/files'
[entries] Prefix the properties in the descriptions with (Node), (Edge) or (Graph) bc3dbbd
The lists of properties at the end of the descriptions marked edge properties with "Edge" but left node properties unmarked, or grouped them in "Nodes: `a`, ...; `b`, ..." items, so the kind of a property was often unclear. Every item now starts with (Node), (Edge) or (Graph), followed by the property names, a colon and the explanation, one item per property or group of properties; qualifications (kinds of nodes, networks) follow the names in parentheses. Properties that were only named got a short explanation, and a few items that named the wrong kind of property were corrected.
diff --git a/entries/wikilinkgraphs/__init__.py b/entries/wikilinkgraphs/__init__.py
index 8532570..54639e4 100644
--- a/entries/wikilinkgraphs/__init__.py
+++ b/entries/wikilinkgraphs/__init__.py
@@ -32,8 +32,8 @@ Related entries: `wikipedia_link` has the current links (including those generat
The networks have the following properties:
-- `page_id`: the identifier of the article in its Wikipedia (as assigned by MediaWiki).
-- `title`: the title of the article (at the time of the snapshot)."""
+- (Node) `page_id`: the identifier of the article in its Wikipedia (as assigned by MediaWiki).
+- (Node) `title`: the title of the article (at the time of the snapshot)."""
tags = ['Informational', 'Web graph', 'Unweighted', 'Metadata']
url = ['https://zenodo.org/records/2539424']
citation = [('C. Consonni, D. Laniado, A. Montresor, "WikiLinkGraphs: A Complete, Longitudinal and Multi-Language Dataset of the Wikipedia Link Networks." Proceedings of the International AAAI Conference on Web and Social Media 13, 598-607 (2019)',
[entries] New entry: wikilinkgraphs 06f7009
diff --git a/entries/wikilinkgraphs/__init__.py b/entries/wikilinkgraphs/__init__.py
new file mode 100644
index 0000000..8532570
--- /dev/null
+++ b/entries/wikilinkgraphs/__init__.py
@@ -0,0 +1,110 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import pyarrow
+import pyarrow.csv
+
+from .. import *
+
+title = "Wikipedia article links, yearly snapshots (WikiLinkGraphs, 2001-2018)"
+description = r"""Yearly snapshots of the networks of links between the articles of the nine largest language editions of Wikipedia (as of 2018), German (`de`), English (`en`), Spanish (`es`), French (`fr`), Italian (`it`), Dutch (`nl`), Polish (`pl`), Russian (`ru`) and Swedish (`sv`), from the WikiLinkGraphs dataset of C. Consonni, D. Laniado and A. Montresor. The authors processed the complete edit history dumps of Wikipedia (of 1 March 2018), taking for each page its last revision before 1 March of each year from 2001 to 2018, and extracted the links written in its wikitext (of the form `[[target]]`).
+
+Nodes are pages of the main namespace (articles), and a directed edge from page A to page B means that the snapshot of A contains a link to B (multiple links from A to B count once). Unlike the links recorded by MediaWiki, the links generated by templates are not included, so that the networks contain only the links added intentionally by the editors. Links to redirect pages were resolved to the articles they redirect to (following the history of the redirects); the redirect pages themselves remain as nodes, with a single outgoing edge to their target and no incoming edges (so that the English network of 2018 has 13.6 million nodes, while the English Wikipedia had 5.6 million articles at the time). Links to pages that did not exist at the time of the snapshot ("red links") and to pages outside the main namespace are not included, and only the pages with some link (incoming or outgoing) are nodes of the networks. Self-loops (pages linking to themselves) are present.
+
+The networks are named by the language code and year of the snapshot, from `de-2002` to `sv-2018`. The snapshots of 2001 are left out for all languages except English, and those of 2002 for Italian and Russian, since they contain no links. The networks of the same language share the page identifiers, so that the nodes of the snapshots can be matched.
+
+Related entries: `wikipedia_link` has the current links (including those generated by templates) of 168 language editions, from the database dumps of 2026; `wikipedia_links_2013` has the links of five language editions in 2013, and of the English Wikipedia yearly from 2015 to 2025, from the Laboratory for Web Algorithmics; `wiki_link_dyn` and `nlwiki_link_intervals` have the additions and removals of links over time of the Simple English and Dutch Wikipedias.
+
+The networks have the following properties:
+
+- `page_id`: the identifier of the article in its Wikipedia (as assigned by MediaWiki).
+- `title`: the title of the article (at the time of the snapshot)."""
+tags = ['Informational', 'Web graph', 'Unweighted', 'Metadata']
+url = ['https://zenodo.org/records/2539424']
+citation = [('C. Consonni, D. Laniado, A. Montresor, "WikiLinkGraphs: A Complete, Longitudinal and Multi-Language Dataset of the Wikipedia Link Networks." Proceedings of the International AAAI Conference on Web and Social Media 13, 598-607 (2019)',
+ 'https://doi.org/10.1609/icwsm.v13i01.3257')]
+icon_hash = None
+upstream_license = ('CC BY 4.0', 'https://creativecommons.org/licenses/by/4.0/')
+upstream_prefix = 'https://zenodo.org/records/2539424/files'
+
+LANGS = ["de", "en", "es", "fr", "it", "nl", "pl", "ru", "sv"]
+# The snapshots without any links (only the header line)
+EMPTY = {(l, 2001) for l in LANGS if l != "en"} | {("it", 2002), ("ru", 2002)}
+
+files = [(f"{lang}wiki.wikilink_graph.{year}-03-01.csv.gz", f"{lang}-{year}", None)
+ for lang in LANGS for year in range(2001, 2019)
+ if (lang, year) not in EMPTY]
+
+def fetch_upstream(force=False):
+ return fetch_upstream_files(__name__.split(".")[-1], upstream_prefix, files,
+ force, check=False)
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+ name = __name__.split(".")[-1]
+ for fname, alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ # Tab-separated: page_id_from, page_title_from, page_id_to,
+ # page_title_to (titles without quoting)
+ src, tgt = [], []
+ titles = {}
+ seen = numpy.zeros(0, dtype="bool")
+ with open_upstream_file(name, fname, "rb") as f:
+ reader = pyarrow.csv.open_csv(
+ f,
+ read_options=pyarrow.csv.ReadOptions(block_size=1 << 26),
+ parse_options=pyarrow.csv.ParseOptions(delimiter="\t",
+ quote_char=False),
+ convert_options=pyarrow.csv.ConvertOptions(
+ column_types={"page_id_from": pyarrow.int64(),
+ "page_title_from": pyarrow.string(),
+ "page_id_to": pyarrow.int64(),
+ "page_title_to": pyarrow.string()}))
+ for batch in reader:
+ s = batch.column("page_id_from").to_numpy()
+ t = batch.column("page_id_to").to_numpy()
+ src.append(s)
+ tgt.append(t)
+ # the titles of the pages not seen before
+ m = max(s.max(), t.max()) + 1 if len(s) else 0
+ if m > len(seen):
+ seen = numpy.concatenate((seen, numpy.zeros(m - len(seen) + (1 << 20),
+ dtype="bool")))
+ for ids, col in ((s, "page_title_from"), (t, "page_title_to")):
+ u, idx = numpy.unique(ids, return_index=True)
+ new = ~seen[u]
+ if new.any():
+ seen[u[new]] = True
+ tl = batch.column(col).take(pyarrow.array(idx[new])).to_pylist()
+ titles.update(zip(u[new].tolist(), tl))
+ del seen
+ src = numpy.concatenate(src)
+ tgt = numpy.concatenate(tgt)
+ ids = numpy.array(sorted(titles), dtype="int64")
+ g = Graph(directed=True)
+ g.add_vertex(len(ids))
+ g.add_edge_list(numpy.column_stack((numpy.searchsorted(ids, src),
+ numpy.searchsorted(ids, tgt))))
+ del src, tgt
+ g.vp.page_id = g.new_vp("int64_t", vals=ids)
+ g.vp.title = g.new_vp("string", vals=[titles[i] for i in ids.tolist()])
+ del titles
+ yield alt, g