History of webbase_2001
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] Add the date when each entry was added, and remove icon_hash beff7c9
The dates are those of the commits that first added the entries' files (2020-08-11 for the entries of the initial commit); renamed entries keep the date of their first name. Changed: none
diff --git a/entries/webbase_2001/__init__.py b/entries/webbase_2001/__init__.py
index ebfbf60..cd8396f 100644
--- a/entries/webbase_2001/__init__.py
+++ b/entries/webbase_2001/__init__.py
@@ -27,7 +27,7 @@ url = 'http://data.law.di.unimi.it/webdata/webbase-2001/'
citation = [('J. Hirai, S. Raghavan, H. Garcia-Molina, and A. Paepcke, "WebBase: a repository of Web pages." Computer Networks 33(1-6), 277-293 (2000).', 'https://doi.org/10.1016/S1389-1286(00)00063-3'),
('P. Boldi and S. Vigna, "The WebGraph framework I: compression techniques." Proceedings of the 13th International Conference on World Wide Web (WWW 2004), 595-602 (2004).', 'https://doi.org/10.1145/988672.988752'),
('P. Boldi, M. Rosa, M. Santini, and S. Vigna, "Layered label propagation: a multiresolution coordinate-free ordering for compressing social networks." Proceedings of the 20th International Conference on World Wide Web (WWW 2011), 587-596 (2011).', 'https://doi.org/10.1145/1963405.1963488')]
-icon_hash = '56a85c8e26855e083a2f793a'
+date_added = '2026-10-25'
upstream_license = None
# Too large to process on an ordinary machine (see --skip-huge)
huge = True
[entries] Remove unneeded code from the entries 2c69f19
Unneeded "global files" statements, unused imports and variables, f-strings without placeholders, duplicate keys in foodweb_little_rock (keeping the order and the values used), and the second definition of get_lines() in twitter_higgs. Nothing changes in the networks or descriptions. Changed: none
diff --git a/entries/webbase_2001/__init__.py b/entries/webbase_2001/__init__.py
index e79d75e..ebfbf60 100644
--- a/entries/webbase_2001/__init__.py
+++ b/entries/webbase_2001/__init__.py
@@ -43,7 +43,6 @@ def fetch_upstream(force=False):
@coerce_props()
@annotate()
def parse(alts=None):
- global files
name = __name__.split(".")[-1]
for fnames, alt, fmt in files:
if alts is not None and alt not in alts:
[entries] Use American spelling in the descriptions 802d564
British spellings in the titles and descriptions of 69 entries (e.g. labelled, organisation, millimetres, centres, anonymised, neighbours, analysed, modelled, colour, licence) are replaced by the American ones. Proper names keep their spelling (Otago Harbour, the Centre for Water Systems, the Centre jeunesse de Montréal, the Dutch Ministry of Defence), as do quoted upstream terms, citations, and property names.
diff --git a/entries/webbase_2001/__init__.py b/entries/webbase_2001/__init__.py
index 77cd621..e79d75e 100644
--- a/entries/webbase_2001/__init__.py
+++ b/entries/webbase_2001/__init__.py
@@ -19,7 +19,7 @@
from .. import *
title = "WebBase web graph (2001)"
-description = """A web graph obtained from the 2001 crawl performed by the Stanford WebBase crawler, as distributed by the Laboratory for Web Algorithmics (LAW). Nodes are web pages (URLs), and a directed edge from one page to another indicates a hyperlink between them. The data provided by WebBase were filtered by LAW to eliminate invalid links and to normalise URLs. The graph has 118,142,155 pages and 1,019,903,190 links, and was used, together with `uk-2002`, in the experiments of the WebGraph compression papers.
+description = """A web graph obtained from the 2001 crawl performed by the Stanford WebBase crawler, as distributed by the Laboratory for Web Algorithmics (LAW). Nodes are web pages (URLs), and a directed edge from one page to another indicates a hyperlink between them. The data provided by WebBase were filtered by LAW to eliminate invalid links and to normalize URLs. The graph has 118,142,155 pages and 1,019,903,190 links, and was used, together with `uk-2002`, in the experiments of the WebGraph compression papers.
The node order is the one of the distributed (LLP-permuted) graph. Vertex property: `url`, the URL of the page (for historical reasons, some URLs contain non-ASCII UTF-8 characters)."""
tags = ['Informational', 'Web graph', 'Unweighted']
[entries] New entry: webbase_2001 552377c
diff --git a/entries/webbase_2001/__init__.py b/entries/webbase_2001/__init__.py
new file mode 100644
index 0000000..77cd621
--- /dev/null
+++ b/entries/webbase_2001/__init__.py
@@ -0,0 +1,52 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from .. import *
+
+title = "WebBase web graph (2001)"
+description = """A web graph obtained from the 2001 crawl performed by the Stanford WebBase crawler, as distributed by the Laboratory for Web Algorithmics (LAW). Nodes are web pages (URLs), and a directed edge from one page to another indicates a hyperlink between them. The data provided by WebBase were filtered by LAW to eliminate invalid links and to normalise URLs. The graph has 118,142,155 pages and 1,019,903,190 links, and was used, together with `uk-2002`, in the experiments of the WebGraph compression papers.
+
+The node order is the one of the distributed (LLP-permuted) graph. Vertex property: `url`, the URL of the page (for historical reasons, some URLs contain non-ASCII UTF-8 characters)."""
+tags = ['Informational', 'Web graph', 'Unweighted']
+url = 'http://data.law.di.unimi.it/webdata/webbase-2001/'
+citation = [('J. Hirai, S. Raghavan, H. Garcia-Molina, and A. Paepcke, "WebBase: a repository of Web pages." Computer Networks 33(1-6), 277-293 (2000).', 'https://doi.org/10.1016/S1389-1286(00)00063-3'),
+ ('P. Boldi and S. Vigna, "The WebGraph framework I: compression techniques." Proceedings of the 13th International Conference on World Wide Web (WWW 2004), 595-602 (2004).', 'https://doi.org/10.1145/988672.988752'),
+ ('P. Boldi, M. Rosa, M. Santini, and S. Vigna, "Layered label propagation: a multiresolution coordinate-free ordering for compressing social networks." Proceedings of the 20th International Conference on World Wide Web (WWW 2011), 587-596 (2011).', 'https://doi.org/10.1145/1963405.1963488')]
+icon_hash = '56a85c8e26855e083a2f793a'
+upstream_license = None
+# Too large to process on an ordinary machine (see --skip-huge)
+huge = True
+upstream_prefix = 'http://data.law.di.unimi.it/webdata'
+files = [((f'{n}/{n}.graph', f'{n}/{n}.properties', f'{n}/{n}.urls.gz'), None, None)
+ for n in ["webbase-2001"]]
+
+def fetch_upstream(force=False):
+ return fetch_upstream_files(__name__.split(".")[-1], upstream_prefix, files,
+ force)
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+ global files
+ name = __name__.split(".")[-1]
+ for fnames, alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ g = law_graph(name, fnames, directed=True, label="url")
+ yield alt, g