logo Netzschleuder network catalog, repository, and centrifuge

History of reddit_hyperlinks

The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.

[entries] New entry: reddit_hyperlinks by Tiago de Paula Peixoto 35ab732
diff --git a/entries/reddit_hyperlinks/__init__.py b/entries/reddit_hyperlinks/__init__.py
new file mode 100644
index 0000000..96d7ba6
--- /dev/null
+++ b/entries/reddit_hyperlinks/__init__.py
@@ -0,0 +1,109 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from .. import *
+
+import datetime
+import zoneinfo
+
+title = "Reddit subreddit hyperlinks (2014-2017)"
+description = r"""Hyperlinks between communities (subreddits) of the discussion website Reddit, extracted by S. Kumar, W. L. Hamilton, J. Leskovec and D. Jurafsky from publicly available Reddit data of January 2014 to April 2017, to study conflicts between communities. Nodes are subreddits, and a directed edge $i \to j$ is a post in subreddit $i$ that contains a hyperlink to a post in subreddit $j$. Each hyperlink is labeled with the sentiment of the source post towards the target post, as positive (or neutral) or negative, which the authors obtained with a text classifier trained on manually labeled examples. A hyperlink can appear in the title or in the body of the post; upstream, these are two separate files, combined here into a single multigraph (a post that links to several subreddits gives several edges). The SNAP page gives 55,863 nodes and 858,490 edges, but the two files contain 858,488 hyperlinks (from 831,014 distinct posts) among 67,180 subreddits; there are no self-loops.
+
+Each hyperlink also has a vector of 86 numeric properties of the text of the source post (e.g. numbers of characters and words, fractions of special characters, sentiment scores computed with VADER, and frequencies of LIWC word categories). There are two networks, with the same nodes and edges:
+
+- `hyperlinks`: the hyperlinks, without the text properties.
+- `hyperlinks_text`: the hyperlinks with the text properties, which make its files much larger.
+
+The subreddit embeddings, also provided by the authors, are not included.
+
+The networks have the following properties:
+
+- `name`: the name of the subreddit.
+- Edge `post_id`: the identifier of the source post.
+- Edge `time`: the time of the post, as Unix time. The time zone of the upstream timestamps is not documented; they are read as US Pacific time, since they then span exactly the period stated by the authors (read as UTC, the first ones would fall on December 31, 2013, from 16:20 on, and the last one on April 30, 2017, at 16:58).
+- Edge `sign`: the sentiment of the source post towards the target post (+1 for positive or neutral, -1 for negative).
+- Edge `in_title`: whether the hyperlink is in the title of the post (otherwise it is in its body).
+- Edge `text_properties` (`hyperlinks_text` only): the 86 properties of the text of the source post, in the order given by the graph property `text_property_names` (as listed by the source, with "unique works" corrected to "unique words")."""
+tags = ['Social', 'Online', 'Communication', 'Signed', 'Multigraph', 'Timestamps']
+url = 'https://snap.stanford.edu/data/soc-RedditHyperlinks.html'
+citation = [('S. Kumar, W. L. Hamilton, J. Leskovec, and D. Jurafsky, "Community Interaction and Conflict on the Web." Proceedings of the 2018 World Wide Web Conference (WWW), 933-943 (2018).', 'https://doi.org/10.1145/3178876.3186141')]
+icon_hash = None
+upstream_license = None
+upstream_prefix = 'https://snap.stanford.edu/data'
+files = [(('soc-redditHyperlinks-body.tsv', 'soc-redditHyperlinks-title.tsv'), 'hyperlinks', None),
+         (('soc-redditHyperlinks-body.tsv', 'soc-redditHyperlinks-title.tsv'), 'hyperlinks_text', None)]
+
+# The names of the 86 text properties of the posts (as listed by the source)
+TEXT_PROPERTY_NAMES = ['Number of characters', 'Number of characters without counting white space', 'Fraction of alphabetical characters', 'Fraction of digits', 'Fraction of uppercase characters', 'Fraction of white spaces', 'Fraction of special characters, such as comma, exclamation mark, etc.', 'Number of words', 'Number of unique words', 'Number of long words (at least 6 characters)', 'Average word length', 'Number of unique stopwords', 'Fraction of stopwords', 'Number of sentences', 'Number of long sentences (at least 10 words)', 'Average number of characters per sentence', 'Average number of words per sentence', 'Automated readability index', 'Positive sentiment calculated by VADER', 'Negative sentiment calculated by VADER', 'Compound sentiment calculated by VADER', 'LIWC_Funct', 'LIWC_Pronoun', 'LIWC_Ppron', 'LIWC_I', 'LIWC_We', 'LIWC_You', 'LIWC_SheHe', 'LIWC_They', 'LIWC_Ipron', 'LIWC_Article', 'LIWC_Verbs', 'LIWC_AuxVb', 'LIWC_Past', 'LIWC_Present', 'LIWC_Future', 'LIWC_Adverbs', 'LIWC_Prep', 'LIWC_Conj', 'LIWC_Negate', 'LIWC_Quant', 'LIWC_Numbers', 'LIWC_Swear', 'LIWC_Social', 'LIWC_Family', 'LIWC_Friends', 'LIWC_Humans', 'LIWC_Affect', 'LIWC_Posemo', 'LIWC_Negemo', 'LIWC_Anx', 'LIWC_Anger', 'LIWC_Sad', 'LIWC_CogMech', 'LIWC_Insight', 'LIWC_Cause', 'LIWC_Discrep', 'LIWC_Tentat', 'LIWC_Certain', 'LIWC_Inhib', 'LIWC_Incl', 'LIWC_Excl', 'LIWC_Percept', 'LIWC_See', 'LIWC_Hear', 'LIWC_Feel', 'LIWC_Bio', 'LIWC_Body', 'LIWC_Health', 'LIWC_Sexual', 'LIWC_Ingest', 'LIWC_Relativ', 'LIWC_Motion', 'LIWC_Space', 'LIWC_Time', 'LIWC_Work', 'LIWC_Achiev', 'LIWC_Leisure', 'LIWC_Home', 'LIWC_Money', 'LIWC_Relig', 'LIWC_Death', 'LIWC_Assent', 'LIWC_Dissent', 'LIWC_Nonflu', 'LIWC_Filler']
+
+def fetch_upstream(force=False):
+    return fetch_upstream_files(__name__.split(".")[-1], upstream_prefix, files,
+                                force)
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+    name = __name__.split(".")[-1]
+    for fnames, alt, fmt in files:
+        if alts is not None and alt not in alts:
+            continue
+        with_text = alt == "hyperlinks_text"
+        g = Graph(directed=True)
+        # the timestamps are in US Pacific time (see the description)
+        tz = zoneinfo.ZoneInfo("America/Los_Angeles")
+        vs = {}
+        E = []
+        post_id = []
+        times = []
+        sign = []
+        in_title = []
+        text = []
+        for fn in fnames:
+            title = fn.endswith("-title.tsv")
+            with open_upstream_file(name, fn, "r") as f:
+                header = next(f).rstrip("\n").split("\t")
+                assert header[:5] == ["SOURCE_SUBREDDIT", "TARGET_SUBREDDIT",
+                                      "POST_ID", "TIMESTAMP", "LINK_SENTIMENT"], header
+                for line in f:
+                    s, t, pid, ts, sent, props = line.split("\t", 5)
+                    for x in (s, t):
+                        if x not in vs:
+                            vs[x] = len(vs)
+                    E.append((vs[s], vs[t]))
+                    post_id.append(pid)
+                    times.append(int(datetime.datetime.strptime(ts, "%Y-%m-%d %H:%M:%S").replace(tzinfo=tz).timestamp()))
+                    sign.append(int(sent))
+                    in_title.append(title)
+                    if with_text:
+                        text.append(numpy.array(props.split(","), dtype="float64"))
+        g.add_vertex(len(vs))
+        g.vp.name = g.new_vp("string", vals=list(vs.keys()))
+        g.add_edge_list(E)
+        g.ep.post_id = g.new_ep("string", vals=post_id)
+        g.ep.time = g.new_ep("int64_t", vals=times)
+        g.ep.sign = g.new_ep("int16_t", vals=sign)
+        g.ep.in_title = g.new_ep("bool", vals=in_title)
+        if with_text:
+            assert all(len(x) == len(TEXT_PROPERTY_NAMES) for x in text)
+            g.ep.text_properties = g.new_ep("vector<double>")
+            g.ep.text_properties.set_2d_array(numpy.array(text).T)
+            g.gp.text_property_names = g.new_gp("vector<string>",
+                                                val=TEXT_PROPERTY_NAMES)
+        yield alt, g