History of bitcoin_transactions
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] Use American spelling in the descriptions 802d564
British spellings in the titles and descriptions of 69 entries (e.g. labelled, organisation, millimetres, centres, anonymised, neighbours, analysed, modelled, colour, licence) are replaced by the American ones. Proper names keep their spelling (Otago Harbour, the Centre for Water Systems, the Centre jeunesse de Montréal, the Dutch Ministry of Defence), as do quoted upstream terms, citations, and property names.
diff --git a/entries/bitcoin_transactions/__init__.py b/entries/bitcoin_transactions/__init__.py
index 9b19a43..44986f2 100644
--- a/entries/bitcoin_transactions/__init__.py
+++ b/entries/bitcoin_transactions/__init__.py
@@ -35,7 +35,7 @@ The networks have the following properties:
- Transaction nodes: `block_number`, the height of the block containing the transaction; `time`, the block timestamp (Unix time, in seconds); `is_coinbase`, whether the transaction is a coinbase transaction; `fee`, the transaction fee in satoshi (10^-8 bitcoin), i.e. the total value of the inputs minus that of the outputs; `size`, the size of the transaction in bytes. These properties are -1 (or false) for address nodes.
- Edges: `index`, the position of the input or output in the transaction; `value`, the value of the input or output in satoshi; `script_type`, the type of the script as classified by bitcoin-etl (pubkeyhash, scripthash, multisig, pubkey or nonstandard).
-**Relation to the hypergraphx datasets.** These samples correspond to the datasets bitcoin-2014, bitcoin-2015 and bitcoin-2016 of Kim et al. (2023), which contain the first 1,500,000 transactions of each month as collected by Wu et al. (2022), and to the datasets crypto-bitcoin-201411, crypto-bitcoin-201506 and crypto-bitcoin-201601 of the hypergraphx-data repository, derived from them. Hypergraphx represents each transaction by the set of distinct addresses of its inputs (here: its in-neighbours) and the set of distinct addresses of its outputs (here: its out-neighbours), annotated with the total input value and the block time. Compared with the networks here, the hypergraphx datasets (i) contain no coinbase transactions, and only about 91% (2014), 94% (2015) and 98% (2016) of the other transactions, with no evident selection rule (the omitted transactions are presumably missing from the data of Wu et al.); (ii) end exactly at the 1,500,000th transaction, inside the last block, while here the last block is complete (at most about 1,100 extra transactions); and (iii) give the block times shifted by the offset of Central European time (one hour earlier in the 2014 and 2016 samples, two hours in the 2015 one)."""
+**Relation to the hypergraphx datasets.** These samples correspond to the datasets bitcoin-2014, bitcoin-2015 and bitcoin-2016 of Kim et al. (2023), which contain the first 1,500,000 transactions of each month as collected by Wu et al. (2022), and to the datasets crypto-bitcoin-201411, crypto-bitcoin-201506 and crypto-bitcoin-201601 of the hypergraphx-data repository, derived from them. Hypergraphx represents each transaction by the set of distinct addresses of its inputs (here: its in-neighbors) and the set of distinct addresses of its outputs (here: its out-neighbors), annotated with the total input value and the block time. Compared with the networks here, the hypergraphx datasets (i) contain no coinbase transactions, and only about 91% (2014), 94% (2015) and 98% (2016) of the other transactions, with no evident selection rule (the omitted transactions are presumably missing from the data of Wu et al.); (ii) end exactly at the 1,500,000th transaction, inside the last block, while here the last block is complete (at most about 1,100 extra transactions); and (iii) give the block times shifted by the offset of Central European time (one hour earlier in the 2014 and 2016 samples, two hours in the 2015 one)."""
tags = ['Economic', 'Cryptocurrency', 'Multigraph', 'Timestamps', 'Metadata']
url = ['https://console.cloud.google.com/marketplace/details/bitcoin/crypto-bitcoin',
'https://github.com/blockchain-etl/bitcoin-etl']
[entries] New entry: bitcoin_transactions e6420c5
diff --git a/entries/bitcoin_transactions/__init__.py b/entries/bitcoin_transactions/__init__.py
new file mode 100644
index 0000000..9b19a43
--- /dev/null
+++ b/entries/bitcoin_transactions/__init__.py
@@ -0,0 +1,153 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from .. import *
+
+title = "Bitcoin address-transaction networks (2014-2016)"
+description = """Directed bipartite networks of Bitcoin addresses and the transactions in which they send or receive bitcoins, for three samples of the Bitcoin blockchain: the first 1.5 million transactions recorded in November 2014, June 2015 and January 2016. A Bitcoin transaction spends outputs of earlier transactions (its inputs), each controlled by an address, and creates new outputs, each paid to an address; the newly mined bitcoins enter through the first transaction of each block (the coinbase transaction), which has no inputs. The data are extracted from the public Bitcoin dataset of Google BigQuery (`bigquery-public-data.crypto_bitcoin.transactions`), which is produced from the blockchain with the open-source bitcoin-etl software.
+
+- `2014_11`: blocks 327,939 to 330,624 (2014-11-01 00:03 to 2014-11-18 21:12 UTC), 1,501,123 transactions.
+- `2015_06`: blocks 358,881 to 360,815 (2015-06-01 00:02 to 2015-06-13 23:14 UTC), 1,500,742 transactions.
+- `2016_01`: blocks 391,182 to 392,551 (2016-01-01 00:03 to 2016-01-09 23:15 UTC), 1,500,831 transactions.
+
+Each sample consists of the complete blocks from the first block of the month (by block timestamp, in UTC) to the block that contains the 1,500,000th transaction of the month.
+
+A node is either an address or a transaction. An edge from an address to a transaction is an input of the transaction, spending an output previously paid to the address; an edge from a transaction to an address is an output of the transaction, paying bitcoins to the address. Addresses are included only as they occur in the inputs and outputs of the sampled transactions. An address that occurs in several inputs (or outputs) of the same transaction is connected to it by parallel edges. An output whose script involves several public keys (bare multisignature outputs) is connected to each of them, with the full value of the output on each edge. Outputs whose script does not encode an address (e.g. data-carrying OP_RETURN outputs or nonstandard scripts) are assigned by bitcoin-etl a pseudo-address "nonstandard<hash of the script>", and have script type "nonstandard".
+
+The networks have the following properties:
+
+- All nodes: `is_transaction`, whether the node is a transaction (otherwise an address); `name`, the address or the transaction hash.
+- Transaction nodes: `block_number`, the height of the block containing the transaction; `time`, the block timestamp (Unix time, in seconds); `is_coinbase`, whether the transaction is a coinbase transaction; `fee`, the transaction fee in satoshi (10^-8 bitcoin), i.e. the total value of the inputs minus that of the outputs; `size`, the size of the transaction in bytes. These properties are -1 (or false) for address nodes.
+- Edges: `index`, the position of the input or output in the transaction; `value`, the value of the input or output in satoshi; `script_type`, the type of the script as classified by bitcoin-etl (pubkeyhash, scripthash, multisig, pubkey or nonstandard).
+
+**Relation to the hypergraphx datasets.** These samples correspond to the datasets bitcoin-2014, bitcoin-2015 and bitcoin-2016 of Kim et al. (2023), which contain the first 1,500,000 transactions of each month as collected by Wu et al. (2022), and to the datasets crypto-bitcoin-201411, crypto-bitcoin-201506 and crypto-bitcoin-201601 of the hypergraphx-data repository, derived from them. Hypergraphx represents each transaction by the set of distinct addresses of its inputs (here: its in-neighbours) and the set of distinct addresses of its outputs (here: its out-neighbours), annotated with the total input value and the block time. Compared with the networks here, the hypergraphx datasets (i) contain no coinbase transactions, and only about 91% (2014), 94% (2015) and 98% (2016) of the other transactions, with no evident selection rule (the omitted transactions are presumably missing from the data of Wu et al.); (ii) end exactly at the 1,500,000th transaction, inside the last block, while here the last block is complete (at most about 1,100 extra transactions); and (iii) give the block times shifted by the offset of Central European time (one hour earlier in the 2014 and 2016 samples, two hours in the 2015 one)."""
+tags = ['Economic', 'Cryptocurrency', 'Multigraph', 'Timestamps', 'Metadata']
+url = ['https://console.cloud.google.com/marketplace/details/bitcoin/crypto-bitcoin',
+ 'https://github.com/blockchain-etl/bitcoin-etl']
+citation = [('J. Wu, J. Liu, W. Chen, H. Huang, Z. Zheng, Y. Zhang, "Detecting Mixing Services via Mining Bitcoin Transaction Network With Hybrid Motifs." IEEE Transactions on Systems, Man, and Cybernetics: Systems 52(4), 2237-2249 (2022)', 'https://doi.org/10.1109/TSMC.2021.3049278'),
+ ('S. Kim, M. Choe, J. Yoo, K. Shin, "Reciprocity in directed hypergraphs: measures, findings, and generators." Data Mining and Knowledge Discovery 37, 2330-2388 (2023)', 'https://doi.org/10.1007/s10618-023-00955-3')]
+icon_hash = None
+upstream_license = ('Creative Commons CC0 1.0 Universal', 'https://www.kaggle.com/datasets/bigquery/bitcoin-blockchain')
+upstream_prefix = None
+
+# alt: (partition month, first block, last block)
+windows = {"2014_11": ("2014-11-01", 327939, 330624),
+ "2015_06": ("2015-06-01", 358881, 360815),
+ "2016_01": ("2016-01-01", 391182, 392551)}
+
+files = [(f"{alt}.csv", alt, None) for alt in windows]
+
+# One row per (transaction, input or output, address); the partition filter
+# on block_timestamp_month limits the scanned data to one month.
+SQL = """
+WITH tx AS (
+ SELECT `hash`, block_number, block_timestamp, is_coinbase, fee, size,
+ inputs, outputs
+ FROM `bigquery-public-data.crypto_bitcoin.transactions`
+ WHERE block_timestamp_month = DATE '{month}'
+ AND block_number BETWEEN {first} AND {last}
+)
+SELECT t.`hash` AS tx_hash, t.block_number,
+ UNIX_SECONDS(t.block_timestamp) AS time, t.is_coinbase,
+ CAST(t.fee AS INT64) AS fee, t.size, FALSE AS is_output,
+ i.index, a AS address, CAST(i.value AS INT64) AS value,
+ i.type AS script_type
+FROM tx AS t, UNNEST(t.inputs) AS i, UNNEST(i.addresses) AS a
+UNION ALL
+SELECT t.`hash` AS tx_hash, t.block_number,
+ UNIX_SECONDS(t.block_timestamp) AS time, t.is_coinbase,
+ CAST(t.fee AS INT64) AS fee, t.size, TRUE AS is_output,
+ o.index, a AS address, CAST(o.value AS INT64) AS value,
+ o.type AS script_type
+FROM tx AS t, UNNEST(t.outputs) AS o, UNNEST(o.addresses) AS a
+"""
+
+def fetch_upstream(force=False):
+ # Needs a Google account authorized for BigQuery (see bigquery.py)
+ name = __name__.split(".")[-1]
+ for fname, alt, fmt in files:
+ month, first, last = windows[alt]
+ fetch_bigquery(name, fname,
+ SQL.format(month=month, first=first, last=last), force)
+
+def to_bool(x):
+ return x.lower() in ("true", "1")
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+ global files
+ name = __name__.split(".")[-1]
+ for fname, alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ vid = {}
+ txs = {} # vertex -> (block_number, time, coinbase, fee, size)
+ src, tgt, index, value, stype = [], [], [], [], []
+ with open_upstream_file(name, fname, "r") as f:
+ reader = csv.reader(f)
+ header = next(reader)
+ c = {k: i for i, k in enumerate(header)}
+ for row in reader:
+ h = row[c["tx_hash"]]
+ t = vid.get(h)
+ if t is None:
+ t = vid[h] = len(vid)
+ txs[t] = (int(row[c["block_number"]]), int(row[c["time"]]),
+ to_bool(row[c["is_coinbase"]]),
+ int(row[c["fee"]] or -1),
+ int(row[c["size"]] or -1))
+ addr = row[c["address"]]
+ a = vid.get(addr)
+ if a is None:
+ a = vid[addr] = len(vid)
+ if to_bool(row[c["is_output"]]):
+ src.append(t)
+ tgt.append(a)
+ else:
+ src.append(a)
+ tgt.append(t)
+ index.append(int(row[c["index"]]))
+ value.append(int(row[c["value"]] or -1))
+ stype.append(row[c["script_type"]])
+
+ g = Graph(directed=True)
+ g.add_vertex(len(vid))
+ g.vp.name = g.new_vp("string", vals=vid.keys())
+ del vid
+ g.vp.is_transaction = g.new_vp("bool")
+ for k in ["block_number", "time", "fee", "size"]:
+ g.vp[k] = g.new_vp("int64_t", val=-1)
+ g.vp.is_coinbase = g.new_vp("bool")
+ tv = numpy.fromiter(txs.keys(), dtype="int64", count=len(txs))
+ props = numpy.array(list(txs.values()), dtype="int64")
+ del txs
+ g.vp.is_transaction.a[tv] = True
+ g.vp.block_number.a[tv] = props[:, 0]
+ g.vp.time.a[tv] = props[:, 1]
+ g.vp.is_coinbase.a[tv] = props[:, 2]
+ g.vp.fee.a[tv] = props[:, 3]
+ g.vp.size.a[tv] = props[:, 4]
+ g.add_edge_list(numpy.column_stack((src, tgt)))
+ g.ep.index = g.new_ep("int32_t")
+ g.ep.index.a = index
+ g.ep.value = g.new_ep("int64_t")
+ g.ep.value.a = value
+ g.ep.script_type = g.new_ep("string", vals=stype)
+ yield alt, g