History of community_notes
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] Add the date when each entry was added, and remove icon_hash beff7c9
The dates are those of the commits that first added the entries' files (2020-08-11 for the entries of the initial commit); renamed entries keep the date of their first name. Changed: none
diff --git a/entries/community_notes/__init__.py b/entries/community_notes/__init__.py
index 07dfa58..8406615 100644
--- a/entries/community_notes/__init__.py
+++ b/entries/community_notes/__init__.py
@@ -69,7 +69,7 @@ citation = [('S. Wojcik, S. Hilgard, N. Judd, D. Mocanu, S. Ragain, M. B. F. Hun
# "publicly available so that anyone has free access to analyze the data"
# (download page); no license is stated
upstream_license = None
-icon_hash = None
+date_added = '2026-10-31'
# The data are published every day under a dated path, and each snapshot is
# kept online for only a few days, so there is no permanent snapshot: the
[entries] Prefix the properties in the descriptions with (Node), (Edge) or (Graph) bc3dbbd
The lists of properties at the end of the descriptions marked edge properties with "Edge" but left node properties unmarked, or grouped them in "Nodes: `a`, ...; `b`, ..." items, so the kind of a property was often unclear. Every item now starts with (Node), (Edge) or (Graph), followed by the property names, a colon and the explanation, one item per property or group of properties; qualifications (kinds of nodes, networks) follow the names in parentheses. Properties that were only named got a short explanation, and a few items that named the wrong kind of property were corrected.
diff --git a/entries/community_notes/__init__.py b/entries/community_notes/__init__.py
index e2fe2e8..07dfa58 100644
--- a/entries/community_notes/__init__.py
+++ b/entries/community_notes/__init__.py
@@ -33,11 +33,34 @@ The data are published as cumulative snapshots, which are kept online for only a
The networks have the following properties:
-- Nodes: `is_note`, whether the node is a note (otherwise a contributor); `has_record`, whether the note (or contributor) is in the file of notes (or of enrollments).
-- Contributors: `participant_id`, the pseudonymous id of the contributor (empty for notes); `enrollment_state` (e.g. `newUser`, `earnedIn`, `atRisk`, `earnedOutNoAcknowledge`); `successful_rating_needed_to_earn_in`; `time_last_state_change` and `time_last_earn_out` (Unix time; small values such as 1 or 0, as in the data, mean never; -1 if unknown); `modeling_population` (e.g. `CORE`, `EXPANSION`); `modeling_group` (NaN if none); `number_of_times_earned_out`.
-- Notes: `note_id`, the id of the note (-1 for contributors); `author_id`, the participant id of its author; `created`, its time of creation (Unix time); `tweet_id`, the id of the post it is about; `classification`, `MISINFORMED_OR_POTENTIALLY_MISLEADING` or `NOT_MISLEADING`; `believable`, `harmful` and `validation_difficulty`, answers of the author in the first version of the form; `summary`, the text of the note, as in the data (line breaks replaced by spaces, some characters HTML-escaped, e.g. `&`); the reasons checked by the author, one property each (1 if checked, 0 if not, -1 if unknown): `misleading_other`, `misleading_factual_error`, `misleading_manipulated_media`, `misleading_outdated_information`, `misleading_missing_important_context`, `misleading_unverified_claim_as_fact`, `misleading_satire`, `not_misleading_other`, `not_misleading_factually_correct`, `not_misleading_outdated_but_not_when_written`, `not_misleading_clearly_satire` and `not_misleading_personal_opinion`; `trustworthy_sources`, whether the author considers the sources trustworthy; `is_media_note` and `is_collaborative_note`; and from the status history: `current_status` (`CURRENTLY_RATED_HELPFUL`, `CURRENTLY_RATED_NOT_HELPFUL` or `NEEDS_MORE_RATINGS`), `time_current_status`, `first_non_nmr_status` and `time_first_non_nmr_status`, the first status other than "needs more ratings", `locked_status` and `time_status_lock`, `current_decided_by`, the scoring model that decided the status (times in Unix time, -1 if none).
-- Edges of `ratings`: `time`, the time of the rating (Unix time); `version`, the version of the rating form (1 or 2); `helpfulness`, the rating, 2 (helpful), 1 (somewhat helpful) or 0 (not helpful), from the `helpful` and `notHelpful` answers in version 1; `reasons`, the reasons given, as a bit mask (bits 0 to 23: `agree`, `disagree`, `helpfulOther`, `helpfulInformative`, `helpfulClear`, `helpfulEmpathetic`, `helpfulGoodSources`, `helpfulUniqueContext`, `helpfulAddressesClaim`, `helpfulImportantContext`, `helpfulUnbiasedLanguage`, `notHelpfulOther`, `notHelpfulIncorrect`, `notHelpfulSourcesMissingOrUnreliable`, `notHelpfulOpinionSpeculationOrBias`, `notHelpfulMissingKeyPoints`, `notHelpfulOutdated`, `notHelpfulHardToUnderstand`, `notHelpfulArgumentativeOrBiased`, `notHelpfulOffTopic`, `notHelpfulSpamHarassmentOrAbuse`, `notHelpfulIrrelevantSources`, `notHelpfulOpinionSpeculation`, `notHelpfulNoteNotNeeded`).
-- Edges of `authorship`: `time`, the time of creation of the note (Unix time)."""
+- (Node) `is_note`: whether the node is a note (otherwise a contributor).
+- (Node) `has_record`: whether the note (or contributor) is in the file of notes (or of enrollments).
+- (Node) `participant_id` (contributors): the pseudonymous id of the contributor (empty for notes).
+- (Node) `enrollment_state` (contributors): the enrollment state (e.g. `newUser`, `earnedIn`, `atRisk`, `earnedOutNoAcknowledge`).
+- (Node) `successful_rating_needed_to_earn_in` (contributors): the number of successful ratings needed to earn in.
+- (Node) `time_last_state_change` and `time_last_earn_out` (contributors): the times of the last state change and of the last earn-out (Unix time; small values such as 1 or 0, as in the data, mean never; -1 if unknown).
+- (Node) `modeling_population` (contributors): the modeling population (e.g. `CORE`, `EXPANSION`).
+- (Node) `modeling_group` (contributors): the modeling group (NaN if none).
+- (Node) `number_of_times_earned_out` (contributors): the number of times the contributor earned out.
+- (Node) `note_id` (notes): the id of the note (-1 for contributors).
+- (Node) `author_id` (notes): the participant id of its author.
+- (Node) `created` (notes): its time of creation (Unix time).
+- (Node) `tweet_id` (notes): the id of the post it is about.
+- (Node) `classification` (notes): `MISINFORMED_OR_POTENTIALLY_MISLEADING` or `NOT_MISLEADING`.
+- (Node) `believable`, `harmful` and `validation_difficulty` (notes): answers of the author in the first version of the form.
+- (Node) `summary` (notes): the text of the note, as in the data (line breaks replaced by spaces, some characters HTML-escaped, e.g. `&`).
+- (Node) `misleading_other`, `misleading_factual_error`, `misleading_manipulated_media`, `misleading_outdated_information`, `misleading_missing_important_context`, `misleading_unverified_claim_as_fact`, `misleading_satire`, `not_misleading_other`, `not_misleading_factually_correct`, `not_misleading_outdated_but_not_when_written`, `not_misleading_clearly_satire` and `not_misleading_personal_opinion` (notes): the reasons checked by the author, one property each (1 if checked, 0 if not, -1 if unknown).
+- (Node) `trustworthy_sources` (notes): whether the author considers the sources trustworthy.
+- (Node) `is_media_note` and `is_collaborative_note` (notes): whether the note is a media note, and whether it is a collaborative note.
+- (Node) `current_status` and `time_current_status` (notes, from the status history): the current status (`CURRENTLY_RATED_HELPFUL`, `CURRENTLY_RATED_NOT_HELPFUL` or `NEEDS_MORE_RATINGS`) and its time (Unix time, -1 if none).
+- (Node) `first_non_nmr_status` and `time_first_non_nmr_status` (notes, from the status history): the first status other than "needs more ratings", and its time (Unix time, -1 if none).
+- (Node) `locked_status` and `time_status_lock` (notes, from the status history): the locked status and the time of the lock (Unix time, -1 if none).
+- (Node) `current_decided_by` (notes, from the status history): the scoring model that decided the status.
+- (Edge) `time` (`ratings`): the time of the rating (Unix time).
+- (Edge) `version` (`ratings`): the version of the rating form (1 or 2).
+- (Edge) `helpfulness` (`ratings`): the rating, 2 (helpful), 1 (somewhat helpful) or 0 (not helpful), from the `helpful` and `notHelpful` answers in version 1.
+- (Edge) `reasons` (`ratings`): the reasons given, as a bit mask (bits 0 to 23: `agree`, `disagree`, `helpfulOther`, `helpfulInformative`, `helpfulClear`, `helpfulEmpathetic`, `helpfulGoodSources`, `helpfulUniqueContext`, `helpfulAddressesClaim`, `helpfulImportantContext`, `helpfulUnbiasedLanguage`, `notHelpfulOther`, `notHelpfulIncorrect`, `notHelpfulSourcesMissingOrUnreliable`, `notHelpfulOpinionSpeculationOrBias`, `notHelpfulMissingKeyPoints`, `notHelpfulOutdated`, `notHelpfulHardToUnderstand`, `notHelpfulArgumentativeOrBiased`, `notHelpfulOffTopic`, `notHelpfulSpamHarassmentOrAbuse`, `notHelpfulIrrelevantSources`, `notHelpfulOpinionSpeculation`, `notHelpfulNoteNotNeeded`).
+- (Edge) `time` (`authorship`): the time of creation of the note (Unix time)."""
tags = ['Social', 'Online', 'Informational', 'Bipartite', 'Unweighted', 'Timestamps', 'Metadata']
url = ['https://communitynotes.x.com/guide/en/under-the-hood/download-data',
'https://ton.twimg.com/birdwatch-public-data/']
[entries] Remove unneeded code from the entries 2c69f19
Unneeded "global files" statements, unused imports and variables, f-strings without placeholders, duplicate keys in foodweb_little_rock (keeping the order and the values used), and the second definition of get_lines() in twitter_higgs. Nothing changes in the networks or descriptions. Changed: none
diff --git a/entries/community_notes/__init__.py b/entries/community_notes/__init__.py
index 2b4fc8d..e2fe2e8 100644
--- a/entries/community_notes/__init__.py
+++ b/entries/community_notes/__init__.py
@@ -201,7 +201,6 @@ def scol(batch, c):
@coerce_props()
@annotate()
def parse(alts=None):
- global files
import pyarrow
name = __name__.split(".")[-1]
with open_upstream_file(name, DATE_FILE, "r") as f:
[entries] New entry: community_notes 06fbb14
diff --git a/entries/community_notes/__init__.py b/entries/community_notes/__init__.py
new file mode 100644
index 0000000..2b4fc8d
--- /dev/null
+++ b/entries/community_notes/__init__.py
@@ -0,0 +1,380 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from .. import *
+import process_entry
+import datetime as dt
+import re
+
+title = "X Community Notes ratings (2021-2026)"
+description = """The ratings of the Community Notes of X (formerly Twitter; the program was called Birdwatch until 2022), the crowd-sourced fact-checking system in which enrolled contributors write notes that add context to posts, and rate the notes of others as helpful or not; a note is shown to all users when its ratings come from contributors with diverse perspectives, as determined by a matrix-factorization ("bridging") algorithm. X publishes all notes, ratings, note statuses and contributor enrollments every day, with pseudonymous contributor ids.
+
+The entry contains two bipartite networks of contributors and notes, which have the same nodes (all contributors and notes of the data):
+
+- `ratings`: an edge connects a contributor to each note they rated, with the rating (helpful, somewhat helpful, or not helpful) and the reasons given.
+- `authorship`: an edge connects each note to its author.
+
+The data are published as cumulative snapshots, which are kept online for only a few days (deleted notes and ratings disappear from them); the networks are built from the most recent snapshot at the time of the download, whose date is stored as the network property `snapshot_date` (the snapshot of 2026-10-30 had five files of notes and nine files of ratings, the first of 5.6 GB uncompressed). Left out are the ids of the posts on which ratings were made (for notes matching several posts), the source buckets of the ratings and the suggestions of the raters. Notes and contributors that are referenced but missing from the files of notes and of enrollments (e.g. deleted notes) are included without attributes (`has_record` false).
+
+The networks have the following properties:
+
+- Nodes: `is_note`, whether the node is a note (otherwise a contributor); `has_record`, whether the note (or contributor) is in the file of notes (or of enrollments).
+- Contributors: `participant_id`, the pseudonymous id of the contributor (empty for notes); `enrollment_state` (e.g. `newUser`, `earnedIn`, `atRisk`, `earnedOutNoAcknowledge`); `successful_rating_needed_to_earn_in`; `time_last_state_change` and `time_last_earn_out` (Unix time; small values such as 1 or 0, as in the data, mean never; -1 if unknown); `modeling_population` (e.g. `CORE`, `EXPANSION`); `modeling_group` (NaN if none); `number_of_times_earned_out`.
+- Notes: `note_id`, the id of the note (-1 for contributors); `author_id`, the participant id of its author; `created`, its time of creation (Unix time); `tweet_id`, the id of the post it is about; `classification`, `MISINFORMED_OR_POTENTIALLY_MISLEADING` or `NOT_MISLEADING`; `believable`, `harmful` and `validation_difficulty`, answers of the author in the first version of the form; `summary`, the text of the note, as in the data (line breaks replaced by spaces, some characters HTML-escaped, e.g. `&`); the reasons checked by the author, one property each (1 if checked, 0 if not, -1 if unknown): `misleading_other`, `misleading_factual_error`, `misleading_manipulated_media`, `misleading_outdated_information`, `misleading_missing_important_context`, `misleading_unverified_claim_as_fact`, `misleading_satire`, `not_misleading_other`, `not_misleading_factually_correct`, `not_misleading_outdated_but_not_when_written`, `not_misleading_clearly_satire` and `not_misleading_personal_opinion`; `trustworthy_sources`, whether the author considers the sources trustworthy; `is_media_note` and `is_collaborative_note`; and from the status history: `current_status` (`CURRENTLY_RATED_HELPFUL`, `CURRENTLY_RATED_NOT_HELPFUL` or `NEEDS_MORE_RATINGS`), `time_current_status`, `first_non_nmr_status` and `time_first_non_nmr_status`, the first status other than "needs more ratings", `locked_status` and `time_status_lock`, `current_decided_by`, the scoring model that decided the status (times in Unix time, -1 if none).
+- Edges of `ratings`: `time`, the time of the rating (Unix time); `version`, the version of the rating form (1 or 2); `helpfulness`, the rating, 2 (helpful), 1 (somewhat helpful) or 0 (not helpful), from the `helpful` and `notHelpful` answers in version 1; `reasons`, the reasons given, as a bit mask (bits 0 to 23: `agree`, `disagree`, `helpfulOther`, `helpfulInformative`, `helpfulClear`, `helpfulEmpathetic`, `helpfulGoodSources`, `helpfulUniqueContext`, `helpfulAddressesClaim`, `helpfulImportantContext`, `helpfulUnbiasedLanguage`, `notHelpfulOther`, `notHelpfulIncorrect`, `notHelpfulSourcesMissingOrUnreliable`, `notHelpfulOpinionSpeculationOrBias`, `notHelpfulMissingKeyPoints`, `notHelpfulOutdated`, `notHelpfulHardToUnderstand`, `notHelpfulArgumentativeOrBiased`, `notHelpfulOffTopic`, `notHelpfulSpamHarassmentOrAbuse`, `notHelpfulIrrelevantSources`, `notHelpfulOpinionSpeculation`, `notHelpfulNoteNotNeeded`).
+- Edges of `authorship`: `time`, the time of creation of the note (Unix time)."""
+tags = ['Social', 'Online', 'Informational', 'Bipartite', 'Unweighted', 'Timestamps', 'Metadata']
+url = ['https://communitynotes.x.com/guide/en/under-the-hood/download-data',
+ 'https://ton.twimg.com/birdwatch-public-data/']
+citation = [('S. Wojcik, S. Hilgard, N. Judd, D. Mocanu, S. Ragain, M. B. F. Hunzaker, K. Coleman, J. Baxter, "Birdwatch: Crowd Wisdom and Bridging Algorithms can Inform Understanding and Reduce the Spread of Misinformation." arXiv:2210.15723 (2022)',
+ 'https://arxiv.org/abs/2210.15723')]
+# "publicly available so that anyone has free access to analyze the data"
+# (download page); no license is stated
+upstream_license = None
+icon_hash = None
+
+# The data are published every day under a dated path, and each snapshot is
+# kept online for only a few days, so there is no permanent snapshot: the
+# most recent one is downloaded, and its date is stored in snapshot_date.txt.
+# The files are split in parts (notes-00000.zip, ...), whose number grows
+# over time; `files` lists the first part of each kind (all are fetched).
+data_prefix = 'https://ton.twimg.com/birdwatch-public-data'
+upstream_prefix = None
+KINDS = [("notes", "notes"), ("noteRatings", "ratings"),
+ ("noteStatusHistory", "noteStatusHistory"),
+ ("userEnrollment", "userEnrollment")]
+def part_fname(kind, base, i):
+ return f"{kind}/{base}-{i:05d}.zip"
+files = [(tuple(part_fname(k, b, 0) for k, b in KINDS), "ratings", None),
+ (tuple(part_fname(k, b, 0) for k, b in KINDS), "authorship", None)]
+DATE_FILE = "snapshot_date.txt"
+
+# Number of rows read per file (None for all; set only for checks)
+_check_rows = None
+
+def url_exists(url, tries=4):
+ "Whether the URL exists (HEAD requests, which sometimes time out)."
+ for i in range(tries):
+ try:
+ with open_url(url, method="HEAD", timeout=60):
+ return True
+ except urllib.error.HTTPError as e:
+ if e.code in (403, 404):
+ return False
+ raise
+ except (TimeoutError, OSError):
+ if i == tries - 1:
+ raise
+ time.sleep(10)
+
+def fetch_upstream(force=False):
+ name = __name__.split(".")[-1]
+ if not force and not process_entry._check_health:
+ try:
+ with open_upstream_file(name, DATE_FILE, "r"):
+ return
+ except (FileNotFoundError, IOError):
+ pass
+ today = dt.datetime.now(dt.timezone.utc).date()
+ for back in range(15):
+ day = today - dt.timedelta(days=back)
+ prefix = f"{data_prefix}/{day:%Y/%m/%d}"
+ if url_exists(f"{prefix}/{part_fname('notes', 'notes', 0)}"):
+ break
+ else:
+ raise RuntimeError("no snapshot of the Community Notes data found")
+ if process_entry._check_health:
+ # a snapshot of the last 15 days is online
+ return
+ parts = []
+ for kind, base in KINDS:
+ i = 0
+ while i == 0 or url_exists(f"{prefix}/{part_fname(kind, base, i)}"):
+ parts.append(part_fname(kind, base, i))
+ i += 1
+ fetch_upstream_files(name, prefix, [(tuple(parts), None, None)], force)
+ with open_upstream_file(name, DATE_FILE, "w") as f:
+ f.write(f"{day:%Y-%m-%d}\n")
+
+def read_parts(name, kind, base, columns, types):
+ """Iterate over the record batches (pyarrow) of all parts of a kind."""
+ import pyarrow, pyarrow.csv
+ i = 0
+ while True:
+ fname = f"{part_fname(kind, base, i)}:{base}-{i:05d}.tsv"
+ cm = open_upstream_file(name, fname, "rb")
+ try:
+ f = cm.__enter__()
+ except (FileNotFoundError, KeyError):
+ if i == 0:
+ raise
+ return
+ try:
+ r = pyarrow.csv.open_csv(
+ f, read_options=pyarrow.csv.ReadOptions(use_threads=False,
+ block_size=1 << 26),
+ parse_options=pyarrow.csv.ParseOptions(delimiter="\t",
+ quote_char=False,
+ newlines_in_values=False),
+ convert_options=pyarrow.csv.ConvertOptions(
+ include_columns=columns, column_types=types,
+ strings_can_be_null=False, quoted_strings_can_be_null=False))
+ n = 0
+ for batch in r:
+ if _check_rows is not None and n >= _check_rows:
+ break
+ n += batch.num_rows
+ yield batch
+ finally:
+ cm.__exit__(None, None, None)
+ i += 1
+
+class Index:
+ "Map keys (strings or integers) to consecutive indices."
+ def __init__(self):
+ self.idx = {}
+ def __call__(self, keys):
+ import pyarrow, pyarrow.compute as pc
+ keys = pyarrow.array(keys)
+ u = pc.unique(keys)
+ ul = u.to_pylist()
+ ids = numpy.array([self.idx.setdefault(k, len(self.idx)) for k in ul],
+ dtype="int64")
+ return ids[pc.index_in(keys, value_set=u).to_numpy()]
+
+RATING_REASONS = ["agree", "disagree", "helpfulOther", "helpfulInformative",
+ "helpfulClear", "helpfulEmpathetic", "helpfulGoodSources",
+ "helpfulUniqueContext", "helpfulAddressesClaim",
+ "helpfulImportantContext", "helpfulUnbiasedLanguage",
+ "notHelpfulOther", "notHelpfulIncorrect",
+ "notHelpfulSourcesMissingOrUnreliable",
+ "notHelpfulOpinionSpeculationOrBias",
+ "notHelpfulMissingKeyPoints", "notHelpfulOutdated",
+ "notHelpfulHardToUnderstand",
+ "notHelpfulArgumentativeOrBiased", "notHelpfulOffTopic",
+ "notHelpfulSpamHarassmentOrAbuse",
+ "notHelpfulIrrelevantSources", "notHelpfulOpinionSpeculation",
+ "notHelpfulNoteNotNeeded"]
+NOTE_REASONS = ["misleadingOther", "misleadingFactualError",
+ "misleadingManipulatedMedia", "misleadingOutdatedInformation",
+ "misleadingMissingImportantContext",
+ "misleadingUnverifiedClaimAsFact", "misleadingSatire",
+ "notMisleadingOther", "notMisleadingFactuallyCorrect",
+ "notMisleadingOutdatedButNotWhenWritten",
+ "notMisleadingClearlySatire", "notMisleadingPersonalOpinion"]
+
+def snake(c):
+ "Property name of a column (camelCase to snake_case)."
+ return re.sub(r"([A-Z])", lambda m: "_" + m.group(1).lower(), c)
+
+def bitmask(batch, cols):
+ import pyarrow.compute as pc
+ m = numpy.zeros(batch.num_rows, dtype="int64")
+ for i, c in enumerate(cols):
+ x = pc.fill_null(batch.column(c), 0).to_numpy(zero_copy_only=False)
+ m |= (x.astype("int64") == 1).astype("int64") << i
+ return m
+
+def col(batch, c, fill=-1):
+ import pyarrow.compute as pc
+ return pc.fill_null(batch.column(c), fill).to_numpy(zero_copy_only=False)
+
+def scol(batch, c):
+ return batch.column(c).to_pylist()
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+ global files
+ import pyarrow
+ name = __name__.split(".")[-1]
+ with open_upstream_file(name, DATE_FILE, "r") as f:
+ snapshot = f.read().strip()
+ for fnames, alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ people = Index()
+ notes = Index()
+ pprops = {} # participant index -> tuple
+ nprops = {} # note index -> list
+ S, I = pyarrow.string(), pyarrow.int64()
+
+ cols = ["participantId", "enrollmentState",
+ "successfulRatingNeededToEarnIn", "timestampOfLastStateChange",
+ "timestampOfLastEarnOut", "modelingPopulation",
+ "modelingGroup", "numberOfTimesEarnedOut"]
+ types = dict(zip(cols, [S, S, I, I, I, S, pyarrow.float64(), I]))
+ for b in read_parts(name, "userEnrollment", "userEnrollment", cols, types):
+ ids = people(b.column("participantId"))
+ vals = [scol(b, c) for c in cols[1:]]
+ for i, row in zip(ids, zip(*vals)):
+ pprops[i] = row
+
+ cols = (["noteId", "noteAuthorParticipantId", "createdAtMillis",
+ "tweetId", "classification", "believable", "harmful",
+ "validationDifficulty"] + NOTE_REASONS +
+ ["trustworthySources", "summary", "isMediaNote",
+ "isCollaborativeNote"])
+ types = {c: I for c in cols}
+ for c in ["noteAuthorParticipantId", "classification", "believable",
+ "harmful", "validationDifficulty", "summary"]:
+ types[c] = S
+ aedges = []
+ for b in read_parts(name, "notes", "notes", cols, types):
+ ids = notes(b.column("noteId"))
+ authors = people(b.column("noteAuthorParticipantId"))
+ created = col(b, "createdAtMillis") // 1000
+ aedges.append(numpy.column_stack((authors, ids, created)))
+ vals = [scol(b, "noteAuthorParticipantId"), created,
+ col(b, "tweetId"), scol(b, "classification"),
+ scol(b, "believable"), scol(b, "harmful"),
+ scol(b, "validationDifficulty"), scol(b, "summary"),
+ col(b, "trustworthySources"), col(b, "isMediaNote"),
+ col(b, "isCollaborativeNote")]
+ reasons = [col(b, c) for c in NOTE_REASONS]
+ for i, row, rs in zip(ids, zip(*vals), zip(*reasons)):
+ # the status history (below) fills the 7 values after the
+ # first 11
+ nprops[i] = list(row) + ["", -1, "", -1, "", -1, ""] + list(rs)
+
+ cols = ["noteId", "noteAuthorParticipantId",
+ "timestampMillisOfFirstNonNMRStatus", "firstNonNMRStatus",
+ "timestampMillisOfCurrentStatus", "currentStatus",
+ "timestampMillisOfStatusLock", "lockedStatus",
+ "currentDecidedBy"]
+ types = dict(zip(cols, [I, S, I, S, I, S, I, S, S]))
+ for b in read_parts(name, "noteStatusHistory", "noteStatusHistory",
+ cols, types):
+ ids = notes(b.column("noteId"))
+ people(b.column("noteAuthorParticipantId"))
+ vals = [scol(b, "currentStatus"),
+ col(b, "timestampMillisOfCurrentStatus") // 1000,
+ scol(b, "firstNonNMRStatus"),
+ col(b, "timestampMillisOfFirstNonNMRStatus") // 1000,
+ scol(b, "lockedStatus"),
+ col(b, "timestampMillisOfStatusLock") // 1000,
+ scol(b, "currentDecidedBy")]
+ for i, row in zip(ids, zip(*vals)):
+ p = nprops.get(i)
+ if p is not None:
+ p[11:18] = row
+
+ redges = []
+ if alt == "ratings":
+ cols = (["noteId", "raterParticipantId", "createdAtMillis",
+ "version", "helpful", "notHelpful", "helpfulnessLevel"] +
+ RATING_REASONS)
+ types = {c: I for c in cols}
+ types["raterParticipantId"] = S
+ types["helpfulnessLevel"] = S
+ levels = {"HELPFUL": 2, "SOMEWHAT_HELPFUL": 1, "NOT_HELPFUL": 0}
+ for b in read_parts(name, "noteRatings", "ratings", cols, types):
+ n = notes(b.column("noteId"))
+ r = people(b.column("raterParticipantId"))
+ lv = numpy.array([levels.get(x, -1) for x in scol(b, "helpfulnessLevel")],
+ dtype="int64")
+ h, nh = col(b, "helpful", 0), col(b, "notHelpful", 0)
+ lv = numpy.where(lv >= 0, lv,
+ numpy.where(h == 1, 2, numpy.where(nh == 1, 0, -1)))
+ redges.append(numpy.column_stack(
+ (r, n, col(b, "createdAtMillis") // 1000, col(b, "version"),
+ lv, bitmask(b, RATING_REASONS))))
+
+ npeople, nnotes = len(people.idx), len(notes.idx)
+ g = Graph(directed=False)
+ g.add_vertex(npeople + nnotes)
+ g.gp.snapshot_date = g.new_gp("string", val=snapshot)
+ g.vp.is_note = g.new_vp("bool")
+ g.vp.is_note.a[npeople:] = True
+ g.vp.has_record = g.new_vp("bool")
+ for i in pprops:
+ g.vp.has_record.a[i] = True
+ for i in nprops:
+ g.vp.has_record.a[npeople + i] = True
+ g.vp.participant_id = g.new_vp("string")
+ for k, i in people.idx.items():
+ g.vp.participant_id[i] = k
+ pnames = [("enrollment_state", "string", ""),
+ ("successful_rating_needed_to_earn_in", "int64_t", -1),
+ ("time_last_state_change", "int64_t", -1),
+ ("time_last_earn_out", "int64_t", -1),
+ ("modeling_population", "string", ""),
+ ("modeling_group", "double", numpy.nan),
+ ("number_of_times_earned_out", "int64_t", -1)]
+ def fill(names, props, offset):
+ for j, (pn, vt, default) in enumerate(names):
+ p = g.new_vp(vt, val=default)
+ if vt == "string":
+ for i, row in props.items():
+ x = row[j]
+ p[offset + i] = "" if x is None else x
+ else:
+ a = p.a
+ for i, row in props.items():
+ x = row[j]
+ if x is not None:
+ a[offset + i] = x
+ g.vp[pn] = p
+ for i, row in pprops.items():
+ # times in milliseconds
+ pprops[i] = (row[0], row[1],
+ row[2] // 1000 if row[2] is not None and row[2] > 1e11 else row[2],
+ row[3] // 1000 if row[3] is not None and row[3] > 1e11 else row[3],
+ row[4], row[5], row[6])
+ fill(pnames, pprops, 0)
+ del pprops
+ g.vp.note_id = g.new_vp("int64_t", val=-1)
+ for k, i in notes.idx.items():
+ g.vp.note_id.a[npeople + i] = k
+ nnames = [("author_id", "string", ""), ("created", "int64_t", -1),
+ ("tweet_id", "int64_t", -1),
+ ("classification", "string", ""),
+ ("believable", "string", ""), ("harmful", "string", ""),
+ ("validation_difficulty", "string", ""),
+ ("summary", "string", ""),
+ ("trustworthy_sources", "int16_t", -1),
+ ("is_media_note", "int16_t", -1),
+ ("is_collaborative_note", "int16_t", -1),
+ ("current_status", "string", ""),
+ ("time_current_status", "int64_t", -1),
+ ("first_non_nmr_status", "string", ""),
+ ("time_first_non_nmr_status", "int64_t", -1),
+ ("locked_status", "string", ""),
+ ("time_status_lock", "int64_t", -1),
+ ("current_decided_by", "string", "")]
+ nnames += [(snake(c), "int16_t", -1) for c in NOTE_REASONS]
+ fill(nnames, nprops, npeople)
+ del nprops
+
+ g.ep.time = g.new_ep("int64_t")
+ if alt == "ratings":
+ g.ep.version = g.new_ep("int16_t")
+ g.ep.helpfulness = g.new_ep("int16_t")
+ g.ep.reasons = g.new_ep("int64_t")
+ for e in redges:
+ e[:, 1] += npeople
+ g.add_edge_list(e, eprops=[g.ep.time, g.ep.version,
+ g.ep.helpfulness, g.ep.reasons])
+ del redges
+ else:
+ for e in aedges:
+ e[:, 1] += npeople
+ g.add_edge_list(e, eprops=[g.ep.time])
+ del aedges
+ yield alt, g