History of voat
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] New entry: voat 8dad1f3
diff --git a/entries/voat/__init__.py b/entries/voat/__init__.py
new file mode 100644
index 0000000..d5bcd1d
--- /dev/null
+++ b/entries/voat/__init__.py
@@ -0,0 +1,211 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import ast
+import calendar
+import json
+import re
+import time as time_mod
+import zipfile
+
+from .. import *
+
+title = "Voat discussions (2013-2020)"
+description = r"""Networks of the discussions on Voat, a news aggregator and discussion website similar to Reddit, popular with communities banned from other platforms, from its foundation in 2013 until its closure on 25 December 2020. The data were collected by Mekacher and Papasavva from the snapshots of the site in the Internet Archive's Wayback Machine and from the API of the site, and contain 2.4 million submissions and 16.3 million comments in 7,100 subverses (the communities of the site, as subreddits on Reddit), and the profiles of 113,000 users and of the subverses. There are two networks:
+
+- `comments`: each node is a user, and a directed edge from user $i$ to user $j$ is a comment of $i$ in the discussion of a submission of $j$, with one edge per comment (a multigraph). The data do not give the comment to which a comment replies (only its depth in the discussion tree), so that the edges point to the author of the submission. Users commenting on their own submissions give self-loops.
+- `subverses`: a bipartite network of users and subverses, where an edge connects a user to a subverse in which the user posted submissions or comments, or that the user moderated or created (according to the profile of the user), with the numbers of submissions and comments.
+
+The submissions and comments of deleted accounts (with no user name in the data), and those posted anonymously (in the subverses that allowed it, where the user names are replaced by random numbers followed by "?"), have no identifiable author: they are left out of both networks, as are the comments on them in `comments`. Comments on submissions that are not in the data (3,753) are left out of `comments`, which thus has 10.8 million of the 16.3 million comments. All the users with a profile are included in both networks, also those without submissions or comments. The texts of the submissions and comments are not included. The times are given in the data as dates and times of the day, taken here as UTC.
+
+The networks have the following properties:
+
+- `name`: the user name, or the name of the subverse.
+- `registered`: the registration date of the user (from the user's profile), in Unix time (-1 if unknown, and for subverses).
+- `is_user` (`subverses` only): whether the node is a user (otherwise a subverse).
+- `subscribers` (`subverses` only): the number of subscribers of the subverse (-1 if unknown, and for users).
+- `created` (`subverses` only): the creation time of the subverse, in Unix time (-1 if unknown, and for users).
+- `about` (`subverses` only): the description of the subverse (empty for users).
+- Edge `time` (`comments` only): the time of the comment, in Unix time.
+- Edge `upvotes`, `downvotes` (`comments` only): the numbers of upvotes and downvotes of the comment.
+- Edge `depth` (`comments` only): the depth of the comment in the discussion (0 for comments on the submission, 1 for replies to these, etc.; -1 if unknown).
+- Edge `subverse` (`comments` only): the subverse of the discussion.
+- Edge `submissions`, `comments` (`subverses` only): the numbers of submissions and comments of the user in the subverse.
+- Edge `moderator`, `owner` (`subverses` only): whether the user moderated, or created, the subverse."""
+tags = ['Social', 'Online', 'Communication', 'Multigraph', 'Timestamps', 'Temporal', 'Bipartite', 'Weighted', 'Metadata']
+url = 'https://doi.org/10.5281/zenodo.5841668'
+citation = [('A. Mekacher, A. Papasavva, "\'I Can\'t Keep It Up.\' A Dataset from the Defunct Voat.co News Aggregator." Proceedings of the International AAAI Conference on Web and Social Media 16, 1302-1311 (2022)', 'https://doi.org/10.1609/icwsm.v16i1.19382')]
+icon_hash = None
+upstream_license = ('CC BY 4.0', 'https://creativecommons.org/licenses/by/4.0/')
+upstream_prefix = 'https://zenodo.org/records/5841668/files'
+files = [(('voat_dataset.zip',), 'comments', None),
+ (('voat_dataset.zip',), 'subverses', None)]
+
+def fetch_upstream(force=False):
+ return fetch_upstream_files(__name__.split(".")[-1], upstream_prefix, files,
+ force)
+
+_ANON = re.compile(r"\d+\?")
+
+def _user(u):
+ "The user name, or None for deleted accounts and anonymous posts."
+ if not isinstance(u, str) or u == "None" or _ANON.fullmatch(u):
+ return None
+ return u
+
+def _time(date, time="00:00:00"):
+ if not date:
+ return -1
+ return calendar.timegm(time_mod.strptime(f"{date} {time}", "%Y-%m-%d %H:%M:%S"))
+
+def _records(z, kind):
+ for n in sorted(z.namelist()):
+ if (n.startswith(f"voat_dataset/{kind}/") and n.endswith(".ndjson")
+ and "checkpoint" not in n):
+ with z.open(n) as f:
+ for line in f:
+ yield json.loads(line)
+
+def _list(s):
+ "A list of subverses in a profile (written as a Python list)."
+ if isinstance(s, list):
+ return s
+ if not isinstance(s, str) or not s:
+ return []
+ return ast.literal_eval(s)
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+ name = __name__.split(".")[-1]
+ alts_ = [alt for fnames, alt, fmt in files if alts is None or alt in alts]
+ if not alts_:
+ return
+ with open_upstream_file(name, files[0][0][0], "rb", seekable=True) as fz:
+ z = zipfile.ZipFile(fz)
+ users = {}
+ with z.open("voat_dataset/users/user_profiles.ndjson") as f:
+ profiles = [json.loads(l) for l in f]
+ with z.open("voat_dataset/subverses/subverse_profiles.ndjson") as f:
+ sprofiles = [json.loads(l) for l in f]
+ # the submissions: id -> author
+ subs = {}
+ ssubs = [] # (user, subverse) of the submissions
+ for r in _records(z, "submissions"):
+ u = _user(r["user"])
+ subs[r["submission_id"]] = u
+ if u is not None:
+ ssubs.append((u, r["subverse"]))
+ comments = []
+ for r in _records(z, "comments"):
+ u = _user(r["user"])
+ if u is None:
+ continue
+ comments.append((u, subs.get(int(r["root_submission"]), False),
+ _time(r["date"], r["time"]), r["upvotes"],
+ r["downvotes"], r["depth"], r["subverse"]))
+ reg = {}
+ for p in profiles:
+ u = _user(p["user"])
+ if u is not None:
+ reg[u] = _time(p["reg_date"][:10]) if p.get("reg_date") else -1
+
+ for alt in alts_:
+ if alt == "comments":
+ index = {}
+ # the users with a profile first, then the others in order of
+ # appearance
+ for p in profiles:
+ u = _user(p["user"])
+ if u is not None:
+ index.setdefault(u, len(index))
+ edges = []
+ for c in comments:
+ if not c[1]: # deleted or anonymous author, or not found
+ continue
+ edges.append((index.setdefault(c[0], len(index)),
+ index.setdefault(c[1], len(index))) + c[2:6])
+ g = Graph(directed=True)
+ g.add_vertex(len(index))
+ g.vp.name = g.new_vp("string", vals=list(index.keys()))
+ g.vp.registered = g.new_vp("int64_t", vals=[reg.get(u, -1) for u in index])
+ eprops = []
+ for k in ["time", "upvotes", "downvotes", "depth"]:
+ g.ep[k] = g.new_ep("int64_t")
+ eprops.append(g.ep[k])
+ g.add_edge_list(numpy.array(edges, dtype="int64").reshape(-1, 6),
+ eprops=eprops)
+ g.ep.subverse = g.new_ep("string",
+ vals=[c[6] for c in comments if c[1]])
+ else:
+ uindex = {}
+ sindex = {}
+ for p in profiles:
+ u = _user(p["user"])
+ if u is not None:
+ uindex.setdefault(u, len(uindex))
+ for p in sprofiles:
+ sindex.setdefault(p["subverse"], len(sindex))
+ counts = {}
+ def pair(u, s):
+ k = (uindex.setdefault(u, len(uindex)), sindex.setdefault(s, len(sindex)))
+ if k not in counts:
+ counts[k] = [0, 0, False, False]
+ return counts[k]
+ for u, s in ssubs:
+ pair(u, s)[0] += 1
+ for c in comments:
+ pair(c[0], c[6])[1] += 1
+ for p in profiles:
+ u = _user(p["user"])
+ if u is None:
+ continue
+ for s in _list(p.get("moderates")):
+ pair(u, s)[2] = True
+ for s in _list(p.get("owns")):
+ pair(u, s)[3] = True
+ nu = len(uindex)
+ g = Graph(directed=False)
+ g.add_vertex(nu + len(sindex))
+ g.vp.name = g.new_vp("string", vals=list(uindex.keys()) + list(sindex.keys()))
+ g.vp.is_user = g.new_vp("bool")
+ g.vp.is_user.a[:nu] = True
+ g.vp.registered = g.new_vp("int64_t", vals=[reg.get(u, -1) for u in uindex] +
+ [-1] * len(sindex))
+ g.vp.subscribers = g.new_vp("int64_t", val=-1)
+ g.vp.created = g.new_vp("int64_t", val=-1)
+ g.vp.about = g.new_vp("string")
+ for p in sprofiles:
+ v = nu + sindex[p["subverse"]]
+ if p.get("subscriber_count") is not None:
+ g.vp.subscribers[v] = p["subscriber_count"]
+ d = p.get("date_created")
+ if d:
+ g.vp.created[v] = _time(d[:10], d[11:19])
+ g.vp.about[v] = p.get("about") or ""
+ eprops = []
+ for k, t in [("submissions", "int64_t"), ("comments", "int64_t"),
+ ("moderator", "bool"), ("owner", "bool")]:
+ g.ep[k] = g.new_ep(t)
+ eprops.append(g.ep[k])
+ g.add_edge_list(numpy.array([(u, nu + s) + tuple(int(x) for x in c)
+ for (u, s), c in counts.items()],
+ dtype="int64").reshape(-1, 6),
+ eprops=eprops)
+ yield alt, g