logo Netzschleuder network catalog, repository, and centrifuge

History of stackexchange_threads

The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.

[entries] New entry: stackexchange_threads by Tiago de Paula Peixoto 70bc5ca
diff --git a/entries/stackexchange_threads/__init__.py b/entries/stackexchange_threads/__init__.py
new file mode 100644
index 0000000..f4ad088
--- /dev/null
+++ b/entries/stackexchange_threads/__init__.py
@@ -0,0 +1,112 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+
+from .. import *
+
+
+title = "Stack Exchange question threads (2008-2017)"
+description = """Bipartite networks of users and the question threads they took part in, on three question-and-answer sites of the Stack Exchange network, assembled by A. R. Benson and collaborators from the Stack Exchange data dump (archive.org), covering the history of each site up to August 2017. Only the threads in which the question and all its answers were posted within 24 hours are included; each such thread is connected to the user who asked the question and to the users who answered it, and is timestamped with the time of the question.
+
+- `ask_ubuntu`: Ask Ubuntu (askubuntu.com), a site for questions about the Ubuntu operating system. Threads span December 2009 to August 2017.
+- `math_sx`: Mathematics Stack Exchange (math.stackexchange.com). Threads span July 2010 to August 2017.
+- `stack_overflow`: Stack Overflow (stackoverflow.com), a site for programming questions. Threads span August 2008 to August 2017.
+- `stack_overflow_25`: the version of `stack_overflow` distributed by the source restricted to the threads with at most 25 users, which leaves out 13 threads (with 26 to 67 users), and the users who took part only in those. Its file of question ids is empty; the question ids were taken from the full version, whose other threads are the same, in the same order.
+
+The users are anonymous (they are identified only by an index of the source data). A user who both asked and answered a question is connected to its thread once.
+
+The networks have the following properties:
+
+- `is_thread`: whether the node is a thread (otherwise a user).
+- Thread nodes: `question_id`, the post id of the question on its site (so that, e.g., question 1 of `ask_ubuntu` is https://askubuntu.com/questions/1); `time`, the time of the question, in Unix time (seconds since 1970-01-01 UTC). The source data give the times in milliseconds relative to the earliest question; they were converted to absolute times by matching the creation times of a sample of the questions, obtained from the Stack Exchange API (the conversion is accurate to about one second, but for a few questions, e.g. migrated from other sites, the times of the data differ from the creation times of the API by up to a few hours)."""
+tags = ['Social', 'Online', 'Collaboration', 'Unweighted', 'Timestamps', 'Metadata']
+url = ['https://www.cs.cornell.edu/~arb/data/threads-ask-ubuntu/',
+       'https://www.cs.cornell.edu/~arb/data/threads-math-sx/',
+       'https://www.cs.cornell.edu/~arb/data/threads-stack-overflow/']
+citation = [('A. R. Benson, R. Abebe, M. T. Schaub, A. Jadbabaie, J. Kleinberg, "Simplicial closure and higher-order link prediction." Proceedings of the National Academy of Sciences 115(48), E11221-E11230 (2018)', 'https://doi.org/10.1073/pnas.1800683115')]
+icon_hash = None
+upstream_license = ("Creative Commons Attribution-ShareAlike (CC BY-SA; version 3.0 or 4.0, depending on the date of each post)",
+                    "https://stackoverflow.com/help/licensing")
+upstream_prefix = 'gdrive://'
+
+_ids = {"ask_ubuntu": ("1ppdJ7CvF_aJ9GLJmVWgKpNL55YVH8MYh", "threads-ask-ubuntu"),
+        "math_sx": ("1aoNCO5IfY14cIKyTir-qAZl78sgMixhA", "threads-math-sx"),
+        "stack_overflow": ("1aUwsvAhse-5tfbVZ494y9Z3cGm5lhXPX", "threads-stack-overflow-full")}
+
+files = [(tuple(f"{did}:tar.gz:{d}/{d}-{x}.txt"
+                for x in ["nverts", "simplices", "times", "simplex-labels"]),
+          alt, None)
+         for alt, (did, d) in _ids.items()]
+# The version of stack_overflow with at most 25 users per thread, with the
+# nverts and the question ids of the full version
+files.append((tuple(f"1s66SSQpUzCzwaBTJ46av0NixzA-UTP9j:tar.gz:threads-stack-overflow/threads-stack-overflow-{x}.txt"
+                    for x in ["nverts", "simplices", "times"]) +
+              tuple(f"{_ids['stack_overflow'][0]}:tar.gz:threads-stack-overflow-full/threads-stack-overflow-full-{x}.txt"
+                    for x in ["nverts", "simplex-labels"]),
+              "stack_overflow_25", None))
+
+# Unix time (in ms) of time 0 of the source data of each dataset, obtained by
+# comparing the source times of two random samples of 100 threads with the
+# creation dates of their questions from the Stack Exchange API
+# (api.stackexchange.com/2.3/questions); apart from a few outliers, the
+# offsets agree to within about 0.5 s.
+_t0 = {"ask_ubuntu": 1261798600850,       # 2009-12-26
+       "math_sx": 1279653874350,          # 2010-07-20
+       "stack_overflow": 1217594591000,   # 2008-08-01
+       "stack_overflow_25": 1217594591000}
+
+def fetch_upstream(force=False):
+    return fetch_upstream_files(__name__.split(".")[-1], upstream_prefix, files,
+                                force)
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+    global files
+    name = __name__.split(".")[-1]
+    for fnames, alt, fmt in files:
+        if alts is not None and alt not in alts:
+            continue
+        with ExitStack() as stack:
+            fs = [stack.enter_context(open_upstream_file(name, fn, "r")) for fn in fnames]
+            threads, users = read_simplices(fs[0], fs[1])
+            times = read_ints(fs[2])
+            if len(fs) == 5:
+                # the threads of the full version with at most 25 users
+                nverts = numpy.bincount(threads, minlength=len(times))
+                full_nverts = read_ints(fs[3])
+                keep = full_nverts <= 25
+                assert numpy.array_equal(full_nverts[keep], nverts)
+                qids = read_ints(fs[4])[keep]
+                del nverts, full_nverts, keep
+            else:
+                qids = read_ints(fs[3])
+        T = len(times)
+        assert len(qids) == T
+        # the user indices of the source data are not contiguous
+        uids, users = numpy.unique(users, return_inverse=True)
+        N = len(uids)
+        g = bipartite_graph(threads, users, T, N, "is_thread")
+        del threads, users
+        g.vp.question_id = g.new_vp("int64_t")
+        g.vp.question_id.a[N:] = qids
+        g.vp.time = g.new_vp("int64_t")
+        g.vp.time.a[N:] = (_t0[alt] + times) // 1000
+        yield alt, g