History of medicare_providers
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] medicare_providers: read files with invalid UTF-8 sequences 2868e05
Some of the CMS files contain bytes that are not valid UTF-8, on which polars stopped with "invalid utf-8 sequence". They are now read with encoding="utf8-lossy", which replaces such bytes with U+FFFD.
diff --git a/entries/medicare_providers/__init__.py b/entries/medicare_providers/__init__.py
index 3f66d74..6af2ce2 100644
--- a/entries/medicare_providers/__init__.py
+++ b/entries/medicare_providers/__init__.py
@@ -152,7 +152,7 @@ def _parse_part_b(name, fname):
with open_upstream_file(name, fname, "rb") as f:
df = pl.read_csv(f, n_rows=_check_rows, columns=cols,
schema_overrides={c: pl.String for c in cols},
- infer_schema=False)
+ infer_schema=False, encoding="utf8-lossy")
scols = ["HCPCS_Cd", "HCPCS_Desc", "HCPCS_Drug_Ind", "Place_Of_Srvc"] + list(_b_provider_cols)
df = df.with_columns([pl.col(c).fill_null("").str.strip_chars() for c in scols] +
[pl.col("Rndrng_NPI").cast(pl.Int64)])
@@ -212,7 +212,7 @@ def parse(alts=None):
"Tot_Day_Suply": pl.Float64,
"Tot_Drug_Cst": pl.Float64,
"Tot_Benes": pl.Float64},
- infer_schema=False)
+ infer_schema=False, encoding="utf8-lossy")
df = df.with_columns(pl.col(c).fill_null("") for c in
["Brnd_Name", "Gnrc_Name"] + list(_prescriber_cols))
[entries] Add the date when each entry was added, and remove icon_hash beff7c9
The dates are those of the commits that first added the entries' files (2020-08-11 for the entries of the initial commit); renamed entries keep the date of their first name. Changed: none
diff --git a/entries/medicare_providers/__init__.py b/entries/medicare_providers/__init__.py
index 2ca4646..3f66d74 100644
--- a/entries/medicare_providers/__init__.py
+++ b/entries/medicare_providers/__init__.py
@@ -68,7 +68,7 @@ citation = [('Centers for Medicare & Medicaid Services, "Medicare Part D Prescri
'https://data.cms.gov/provider-summary-by-type-of-service/medicare-part-d-prescribers/medicare-part-d-prescribers-by-provider-and-drug'),
('Centers for Medicare & Medicaid Services, "Medicare Physician & Other Practitioners - by Provider and Service." (2013-2024)',
'https://data.cms.gov/provider-summary-by-type-of-service/medicare-physician-other-practitioners/medicare-physician-other-practitioners-by-provider-and-service')]
-icon_hash = None
+date_added = '2026-10-31'
upstream_license = ("Public domain (US government work)", "https://www.usa.gov/government-copyright")
upstream_prefix = 'https://data.cms.gov/sites/default/files'
[entries] Prefix the properties in the descriptions with (Node), (Edge) or (Graph) bc3dbbd
The lists of properties at the end of the descriptions marked edge properties with "Edge" but left node properties unmarked, or grouped them in "Nodes: `a`, ...; `b`, ..." items, so the kind of a property was often unclear. Every item now starts with (Node), (Edge) or (Graph), followed by the property names, a colon and the explanation, one item per property or group of properties; qualifications (kinds of nodes, networks) follow the names in parentheses. Properties that were only named got a short explanation, and a few items that named the wrong kind of property were corrected.
diff --git a/entries/medicare_providers/__init__.py b/entries/medicare_providers/__init__.py
index 919a956..2ca4646 100644
--- a/entries/medicare_providers/__init__.py
+++ b/entries/medicare_providers/__init__.py
@@ -27,40 +27,40 @@ The nodes are the prescribers, identified by their National Provider Identifier
The `part_d_<year>` networks have the following properties:
-- `is_prescriber`: whether the node is a prescriber (otherwise it is a drug).
-- `npi` (prescribers): the National Provider Identifier of the prescriber (0 for drugs).
-- `name`: the last name of the prescriber (or the name of the organization), or the brand name of the drug.
-- `first_name` (prescribers): the first name of the prescriber (empty for drugs and organizations).
-- `city`, `state` (prescribers): the city and the state (two-letter abbreviation of the state or territory, e.g. `PR`, `GU`; `ZZ` for a foreign country, `AA`, `AE`, `AP` for the armed forces) of the prescriber, from the National Plan and Provider Enumeration System (NPPES).
-- `specialty` (prescribers): the type of the prescriber (e.g. `Internal Medicine`, `Nurse Practitioner`, `Dentist`).
-- `specialty_source` (prescribers): the source of `specialty`: `Claim-Specialty` (the specialty codes of the claims of Medicare Part B), or, for the other prescribers (e.g. those who do not bill Medicare Part B), `NPPES-Specialty` or `NPPES-Taxonomy` (the provider taxonomy of the NPPES).
-- `generic_name` (drugs): the generic name of the drug (its active ingredients).
-- Edge `claims`: the number of Part D claims (fills of the prescriptions, including refills).
-- Edge `fills`: the number of standardized 30-day fills (fills of more than 30 days counted as multiples of 30 days).
-- Edge `days_supply`: the total number of days of supply of the fills.
-- Edge `cost`: the total drug cost paid by the Part D plans, the beneficiaries and other payers, in US dollars (ingredient cost, dispensing fee, sales tax and vaccine administration fee).
-- Edge `beneficiaries`: the number of distinct beneficiaries (-1 if suppressed, i.e. fewer than 11).
+- (Node) `is_prescriber`: whether the node is a prescriber (otherwise it is a drug).
+- (Node) `npi` (prescribers): the National Provider Identifier of the prescriber (0 for drugs).
+- (Node) `name`: the last name of the prescriber (or the name of the organization), or the brand name of the drug.
+- (Node) `first_name` (prescribers): the first name of the prescriber (empty for drugs and organizations).
+- (Node) `city`, `state` (prescribers): the city and the state (two-letter abbreviation of the state or territory, e.g. `PR`, `GU`; `ZZ` for a foreign country, `AA`, `AE`, `AP` for the armed forces) of the prescriber, from the National Plan and Provider Enumeration System (NPPES).
+- (Node) `specialty` (prescribers): the type of the prescriber (e.g. `Internal Medicine`, `Nurse Practitioner`, `Dentist`).
+- (Node) `specialty_source` (prescribers): the source of `specialty`: `Claim-Specialty` (the specialty codes of the claims of Medicare Part B), or, for the other prescribers (e.g. those who do not bill Medicare Part B), `NPPES-Specialty` or `NPPES-Taxonomy` (the provider taxonomy of the NPPES).
+- (Node) `generic_name` (drugs): the generic name of the drug (its active ingredients).
+- (Edge) `claims`: the number of Part D claims (fills of the prescriptions, including refills).
+- (Edge) `fills`: the number of standardized 30-day fills (fills of more than 30 days counted as multiples of 30 days).
+- (Edge) `days_supply`: the total number of days of supply of the fills.
+- (Edge) `cost`: the total drug cost paid by the Part D plans, the beneficiaries and other payers, in US dollars (ingredient cost, dispensing fee, sales tax and vaccine administration fee).
+- (Edge) `beneficiaries`: the number of distinct beneficiaries (-1 if suppressed, i.e. fewer than 11).
**Part B** (`part_b_<year>`): networks of the providers of services (physicians, other practitioners and organizations) to the beneficiaries of Medicare Part B (the fee-for-service medical insurance) and the services that they provided, from the "Medicare Physician & Other Practitioners - by Provider and Service" data of CMS, compiled from the final-action claims of the Part B fee-for-service beneficiaries of each calendar year. The nodes are the rendering providers, identified by their NPI, and the services, identified by their Healthcare Common Procedure Coding System (HCPCS) code, and an undirected edge connects a provider to a service that they provided during the year, at a given place of service (facility or office, so that a provider and a service can be joined by two edges, one for each place). CMS left out the combinations of provider, service and place of service with 10 or fewer beneficiaries, so that each network is thresholded at this value (the complete data are not public). Only the providers and services that take part in some edge are included.
The `part_b_<year>` networks have the following properties:
-- `is_provider`: whether the node is a provider (otherwise it is a service).
-- `npi` (providers): the National Provider Identifier of the provider (0 for services).
-- `name`: the last name of the provider (or the name of the organization), or the HCPCS code of the service.
-- `first_name`, `middle_initial`, `credentials` (providers): the first name, middle initial and credentials (e.g. `M.D.`) of the provider (empty for organizations).
-- `entity_type` (providers): `I` for individuals, `O` for organizations.
-- `street`, `city`, `state`, `zip5`, `country` (providers): the address of the provider, from the NPPES.
-- `ruca` (providers): the Rural-Urban Commuting Area code of the ZIP code of the provider (as given, e.g. `1` for a metropolitan core; empty if unknown).
-- `specialty` (providers): the type of the provider (from the specialty codes of their claims, e.g. `Internal Medicine`).
-- `participating` (providers): whether the provider participates in Medicare (accepts the Medicare-approved amount as full payment; `Y` or `N`).
-- `hcpcs_code`, `description` (services): the HCPCS code and its description.
-- `is_drug` (services): whether the service is a drug of the Part B drug average sales price file (e.g. injections).
-- Edge `place_of_service`: `F` (facility, e.g. hospital) or `O` (office, non-facility).
-- Edge `beneficiaries`: the number of distinct beneficiaries.
-- Edge `services`: the number of services provided.
-- Edge `beneficiary_days`: the number of distinct beneficiary per day services.
-- Edge `avg_submitted_charge`, `avg_allowed_amount`, `avg_payment`, `avg_standardized_payment`: the average charge submitted by the provider, the average amount allowed by Medicare, the average amount paid by Medicare, and the average payment standardized for geographic differences, per service, in US dollars."""
+- (Node) `is_provider`: whether the node is a provider (otherwise it is a service).
+- (Node) `npi` (providers): the National Provider Identifier of the provider (0 for services).
+- (Node) `name`: the last name of the provider (or the name of the organization), or the HCPCS code of the service.
+- (Node) `first_name`, `middle_initial`, `credentials` (providers): the first name, middle initial and credentials (e.g. `M.D.`) of the provider (empty for organizations).
+- (Node) `entity_type` (providers): `I` for individuals, `O` for organizations.
+- (Node) `street`, `city`, `state`, `zip5`, `country` (providers): the address of the provider, from the NPPES.
+- (Node) `ruca` (providers): the Rural-Urban Commuting Area code of the ZIP code of the provider (as given, e.g. `1` for a metropolitan core; empty if unknown).
+- (Node) `specialty` (providers): the type of the provider (from the specialty codes of their claims, e.g. `Internal Medicine`).
+- (Node) `participating` (providers): whether the provider participates in Medicare (accepts the Medicare-approved amount as full payment; `Y` or `N`).
+- (Node) `hcpcs_code`, `description` (services): the HCPCS code and its description.
+- (Node) `is_drug` (services): whether the service is a drug of the Part B drug average sales price file (e.g. injections).
+- (Edge) `place_of_service`: `F` (facility, e.g. hospital) or `O` (office, non-facility).
+- (Edge) `beneficiaries`: the number of distinct beneficiaries.
+- (Edge) `services`: the number of services provided.
+- (Edge) `beneficiary_days`: the number of distinct beneficiary per day services.
+- (Edge) `avg_submitted_charge`, `avg_allowed_amount`, `avg_payment`, `avg_standardized_payment`: the average charge submitted by the provider, the average amount allowed by Medicare, the average amount paid by Medicare, and the average payment standardized for geographic differences, per service, in US dollars."""
tags = ['Economic', 'Public Health', 'Bipartite', 'Multigraph', 'Weighted', 'Metadata']
url = ['https://data.cms.gov/provider-summary-by-type-of-service/medicare-part-d-prescribers/medicare-part-d-prescribers-by-provider-and-drug',
'https://data.cms.gov/provider-summary-by-type-of-service/medicare-physician-other-practitioners/medicare-physician-other-practitioners-by-provider-and-service']
[entries] medicare_part_d: rename to medicare_providers, and add the Part B provider-service networks 47aa96a
The entry had only the Medicare Part D prescriber-drug networks. It now also has, per year (2013 to 2024), the networks of the providers and the services (HCPCS codes) they billed, from "Medicare Physician & Other Practitioners - by Provider and Service", so it is renamed to medicare_providers; the Part D networks are named part_d_<year>, the new ones part_b_<year>. The content of the Part D networks is unchanged.
diff --git a/entries/medicare_providers/__init__.py b/entries/medicare_providers/__init__.py
new file mode 100644
index 0000000..919a956
--- /dev/null
+++ b/entries/medicare_providers/__init__.py
@@ -0,0 +1,255 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from .. import *
+
+title = "Medicare providers and their prescribed drugs and services (2013-2024)"
+description = r"""Bipartite networks of the health care providers of the US Medicare health insurance and the drugs that they prescribed (Medicare Part D) or the services that they provided (Medicare Part B), from data published yearly by the Centers for Medicare & Medicaid Services (CMS), with two families of networks, one network per year of each, from 2013 to 2024: `part_d_<year>` (e.g. `part_d_2024`) and `part_b_<year>` (e.g. `part_b_2024`).
+
+**Part D** (`part_d_<year>`): networks of the prescribers of drugs (physicians and other health care providers) and the drugs they prescribed to the beneficiaries of Medicare Part D, the prescription drug program of the US Medicare health insurance, from the "Medicare Part D Prescribers - by Provider and Drug" data published yearly by the Centers for Medicare & Medicaid Services (CMS). The data were compiled by CMS from the prescription drug event records submitted by the Part D plans (stand-alone prescription drug plans and Medicare Advantage plans), and cover the prescriptions filled in each calendar year, from 2013 to 2024, with one network per year.
+
+The nodes are the prescribers, identified by their National Provider Identifier (NPI), and the drugs, identified by their brand name and generic name (as normalized by CMS; generic drugs have the generic name as brand name), and an undirected edge connects a prescriber to a drug that they prescribed during the year, with the number of claims (prescription fills, including refills), the number of standardized 30-day fills, the total days' supply, the total drug cost, and the number of distinct beneficiaries. CMS left out the combinations of prescriber and drug with fewer than 11 claims in the year, to protect the privacy of the beneficiaries, so that each network is thresholded at this value (the complete data are not public); likewise, the number of beneficiaries is suppressed when it is below 11 (and given as -1 here). Only the prescribers and drugs that take part in some edge are included. The breakdown of the counts for the beneficiaries of 65 years or older (and its suppression flags), given in the source, is not included.
+
+The `part_d_<year>` networks have the following properties:
+
+- `is_prescriber`: whether the node is a prescriber (otherwise it is a drug).
+- `npi` (prescribers): the National Provider Identifier of the prescriber (0 for drugs).
+- `name`: the last name of the prescriber (or the name of the organization), or the brand name of the drug.
+- `first_name` (prescribers): the first name of the prescriber (empty for drugs and organizations).
+- `city`, `state` (prescribers): the city and the state (two-letter abbreviation of the state or territory, e.g. `PR`, `GU`; `ZZ` for a foreign country, `AA`, `AE`, `AP` for the armed forces) of the prescriber, from the National Plan and Provider Enumeration System (NPPES).
+- `specialty` (prescribers): the type of the prescriber (e.g. `Internal Medicine`, `Nurse Practitioner`, `Dentist`).
+- `specialty_source` (prescribers): the source of `specialty`: `Claim-Specialty` (the specialty codes of the claims of Medicare Part B), or, for the other prescribers (e.g. those who do not bill Medicare Part B), `NPPES-Specialty` or `NPPES-Taxonomy` (the provider taxonomy of the NPPES).
+- `generic_name` (drugs): the generic name of the drug (its active ingredients).
+- Edge `claims`: the number of Part D claims (fills of the prescriptions, including refills).
+- Edge `fills`: the number of standardized 30-day fills (fills of more than 30 days counted as multiples of 30 days).
+- Edge `days_supply`: the total number of days of supply of the fills.
+- Edge `cost`: the total drug cost paid by the Part D plans, the beneficiaries and other payers, in US dollars (ingredient cost, dispensing fee, sales tax and vaccine administration fee).
+- Edge `beneficiaries`: the number of distinct beneficiaries (-1 if suppressed, i.e. fewer than 11).
+
+**Part B** (`part_b_<year>`): networks of the providers of services (physicians, other practitioners and organizations) to the beneficiaries of Medicare Part B (the fee-for-service medical insurance) and the services that they provided, from the "Medicare Physician & Other Practitioners - by Provider and Service" data of CMS, compiled from the final-action claims of the Part B fee-for-service beneficiaries of each calendar year. The nodes are the rendering providers, identified by their NPI, and the services, identified by their Healthcare Common Procedure Coding System (HCPCS) code, and an undirected edge connects a provider to a service that they provided during the year, at a given place of service (facility or office, so that a provider and a service can be joined by two edges, one for each place). CMS left out the combinations of provider, service and place of service with 10 or fewer beneficiaries, so that each network is thresholded at this value (the complete data are not public). Only the providers and services that take part in some edge are included.
+
+The `part_b_<year>` networks have the following properties:
+
+- `is_provider`: whether the node is a provider (otherwise it is a service).
+- `npi` (providers): the National Provider Identifier of the provider (0 for services).
+- `name`: the last name of the provider (or the name of the organization), or the HCPCS code of the service.
+- `first_name`, `middle_initial`, `credentials` (providers): the first name, middle initial and credentials (e.g. `M.D.`) of the provider (empty for organizations).
+- `entity_type` (providers): `I` for individuals, `O` for organizations.
+- `street`, `city`, `state`, `zip5`, `country` (providers): the address of the provider, from the NPPES.
+- `ruca` (providers): the Rural-Urban Commuting Area code of the ZIP code of the provider (as given, e.g. `1` for a metropolitan core; empty if unknown).
+- `specialty` (providers): the type of the provider (from the specialty codes of their claims, e.g. `Internal Medicine`).
+- `participating` (providers): whether the provider participates in Medicare (accepts the Medicare-approved amount as full payment; `Y` or `N`).
+- `hcpcs_code`, `description` (services): the HCPCS code and its description.
+- `is_drug` (services): whether the service is a drug of the Part B drug average sales price file (e.g. injections).
+- Edge `place_of_service`: `F` (facility, e.g. hospital) or `O` (office, non-facility).
+- Edge `beneficiaries`: the number of distinct beneficiaries.
+- Edge `services`: the number of services provided.
+- Edge `beneficiary_days`: the number of distinct beneficiary per day services.
+- Edge `avg_submitted_charge`, `avg_allowed_amount`, `avg_payment`, `avg_standardized_payment`: the average charge submitted by the provider, the average amount allowed by Medicare, the average amount paid by Medicare, and the average payment standardized for geographic differences, per service, in US dollars."""
+tags = ['Economic', 'Public Health', 'Bipartite', 'Multigraph', 'Weighted', 'Metadata']
+url = ['https://data.cms.gov/provider-summary-by-type-of-service/medicare-part-d-prescribers/medicare-part-d-prescribers-by-provider-and-drug',
+ 'https://data.cms.gov/provider-summary-by-type-of-service/medicare-physician-other-practitioners/medicare-physician-other-practitioners-by-provider-and-service']
+citation = [('Centers for Medicare & Medicaid Services, "Medicare Part D Prescribers - by Provider and Drug." (2013-2024)',
+ 'https://data.cms.gov/provider-summary-by-type-of-service/medicare-part-d-prescribers/medicare-part-d-prescribers-by-provider-and-drug'),
+ ('Centers for Medicare & Medicaid Services, "Medicare Physician & Other Practitioners - by Provider and Service." (2013-2024)',
+ 'https://data.cms.gov/provider-summary-by-type-of-service/medicare-physician-other-practitioners/medicare-physician-other-practitioners-by-provider-and-service')]
+icon_hash = None
+upstream_license = ("Public domain (US government work)", "https://www.usa.gov/government-copyright")
+
+upstream_prefix = 'https://data.cms.gov/sites/default/files'
+# The files of each data year, as listed in https://data.cms.gov/data.json;
+# each release has its own (dated) path, so that the files are not replaced
+# in place
+_files = {
+ 2013: "2024-05/5fb694b1-2ec5-4e00-8efe-14161bdbdbea/MUP_DPR_RY24_P04_V10_DY13_NPIBN.csv",
+ 2014: "2024-05/06e87540-68a5-4a10-bf9c-a0521dc4ebed/MUP_DPR_RY24_P04_V10_DY14_NPIBN.csv",
+ 2015: "2024-05/bc1caf7f-dcbb-4258-b243-ee3666b6a20b/MUP_DPR_RY24_P04_V10_DY15_NPIBN.csv",
+ 2016: "2024-05/67c457c6-b62d-424f-ad85-2bb8117c928d/MUP_DPR_RY24_P04_V10_DY16_NPIBN.csv",
+ 2017: "2024-05/d9d685a0-dc49-4da5-9416-4cf3e6349296/MUP_DPR_RY24_P04_V10_DY17_NPIBN.csv",
+ 2018: "2024-05/be3019b4-1164-4b40-af5d-cab40847d222/MUP_DPR_RY24_P04_V10_DY18_NPIBN.csv",
+ 2019: "2024-05/129e7c21-d492-425b-be03-d2e59d933ab6/MUP_DPR_RY24_P04_V10_DY19_NPIBN.csv",
+ 2020: "2024-05/75fecc51-c9e8-4904-b570-9da9dc101721/MUP_DPR_RY24_P04_V10_DY20_NPIBN.csv",
+ 2021: "2024-05/43359391-e7fa-40b9-9bd4-5dc295e18712/MUP_DPR_RY24_P04_V10_DY21_NPIBN.csv",
+ 2022: "2024-05/18f82097-61a6-4889-9941-9a0b6ad7523c/MUP_DPR_RY24_P04_V10_DY22_NPIBN.csv",
+ 2023: "2025-04/0d5915ce-002c-4d87-bde8-24ffb08bb6cc/MUP_DPR_RY25_P04_V10_DY23_NPIBN.csv",
+ 2024: "2026-05/0ae165f4-eb44-495d-8cac-67f4571b6b83/MUP_DPR_RY26_P04_V10_DY24_NPIBN.csv",
+}
+# The Part B files ("by Provider and Service") of each data year
+_files_b = {
+ 2013: "2025-11/bf4231f9-ec7f-4189-afc3-ba5d53b8bd12/MUP_PHY_R25_P04_V20_D13_Prov_Svc.csv",
+ 2014: "2025-11/6700f86d-d2e5-4f2d-9dcb-8c30412768ff/MUP_PHY_R25_P04_V20_D14_Prov_Svc.csv",
+ 2015: "2025-11/14954ce3-4c43-43df-97e9-2c0437d7b43c/MUP_PHY_R25_P04_V20_D15_Prov_Svc.csv",
+ 2016: "2025-11/426bf97a-4cb8-47ca-9727-a535d9e8c298/MUP_PHY_R25_P04_V20_D16_Prov_Svc.csv",
+ 2017: "2025-11/4623fb40-781e-4eef-860e-b851cd5d10ea/MUP_PHY_R25_P04_V20_D17_Prov_Svc.csv",
+ 2018: "2025-11/5669eafb-f0b3-4dc5-be6d-abc09b480c2e/MUP_PHY_R25_P04_V20_D18_Prov_Svc.csv",
+ 2019: "2025-11/7befba27-752e-47a8-a76c-6c6d4f74f2e3/MUP_PHY_R25_P04_V20_D19_Prov_Svc.csv",
+ 2020: "2025-11/d22b18cd-7726-4bf5-8e9c-3e4587c589a1/MUP_PHY_R25_P05_V20_D20_Prov_Svc.csv",
+ 2021: "2025-11/bffaf97a-c2ab-4fd7-8718-be90742e3485/MUP_PHY_R25_P05_V20_D21_Prov_Svc.csv",
+ 2022: "2025-11/53fb2bae-4913-48dc-a6d4-d8c025906567/MUP_PHY_R25_P05_V20_D22_Prov_Svc.csv",
+ 2023: "2025-04/e3f823f8-db5b-4cc7-ba04-e7ae92b99757/MUP_PHY_R25_P05_V20_D23_Prov_Svc.csv",
+ 2024: "2026-05/b5ebab5a-f490-418a-9bce-4b9f31419356/PHY_R26_P05_V10_D24_Prov_Svc.csv",
+}
+files = ([(f, f"part_d_{y}", None) for y, f in _files.items()] +
+ [(f, f"part_b_{y}", None) for y, f in _files_b.items()])
+
+def fetch_upstream(force=False):
+ return fetch_upstream_files(__name__.split(".")[-1], upstream_prefix, files,
+ force)
+
+# For checks only: if set, only this many rows are read
+_check_rows = None
+
+_prescriber_cols = {"Prscrbr_Last_Org_Name": "name",
+ "Prscrbr_First_Name": "first_name",
+ "Prscrbr_City": "city",
+ "Prscrbr_State_Abrvtn": "state",
+ "Prscrbr_Type": "specialty",
+ "Prscrbr_Type_Src": "specialty_source"}
+_edge_cols = {"Tot_Clms": "claims", "Tot_30day_Fills": "fills",
+ "Tot_Day_Suply": "days_supply", "Tot_Drug_Cst": "cost",
+ "Tot_Benes": "beneficiaries"}
+
+_b_provider_cols = {"Rndrng_Prvdr_Last_Org_Name": "name",
+ "Rndrng_Prvdr_First_Name": "first_name",
+ "Rndrng_Prvdr_MI": "middle_initial",
+ "Rndrng_Prvdr_Crdntls": "credentials",
+ "Rndrng_Prvdr_Ent_Cd": "entity_type",
+ "Rndrng_Prvdr_St1": "street",
+ "Rndrng_Prvdr_City": "city",
+ "Rndrng_Prvdr_State_Abrvtn": "state",
+ "Rndrng_Prvdr_Zip5": "zip5",
+ "Rndrng_Prvdr_Cntry": "country",
+ "Rndrng_Prvdr_RUCA": "ruca",
+ "Rndrng_Prvdr_Type": "specialty",
+ "Rndrng_Prvdr_Mdcr_Prtcptg_Ind": "participating"}
+_b_edge_cols = {"Place_Of_Srvc": "place_of_service",
+ "Tot_Benes": "beneficiaries", "Tot_Srvcs": "services",
+ "Tot_Bene_Day_Srvcs": "beneficiary_days",
+ "Avg_Sbmtd_Chrg": "avg_submitted_charge",
+ "Avg_Mdcr_Alowd_Amt": "avg_allowed_amount",
+ "Avg_Mdcr_Pymt_Amt": "avg_payment",
+ "Avg_Mdcr_Stdzd_Amt": "avg_standardized_payment"}
+
+def _parse_part_b(name, fname):
+ import polars as pl
+ cols = (["Rndrng_NPI", "HCPCS_Cd", "HCPCS_Desc", "HCPCS_Drug_Ind"] +
+ list(_b_provider_cols) + list(_b_edge_cols))
+ with open_upstream_file(name, fname, "rb") as f:
+ df = pl.read_csv(f, n_rows=_check_rows, columns=cols,
+ schema_overrides={c: pl.String for c in cols},
+ infer_schema=False)
+ scols = ["HCPCS_Cd", "HCPCS_Desc", "HCPCS_Drug_Ind", "Place_Of_Srvc"] + list(_b_provider_cols)
+ df = df.with_columns([pl.col(c).fill_null("").str.strip_chars() for c in scols] +
+ [pl.col("Rndrng_NPI").cast(pl.Int64)])
+ prov = (df.select(["Rndrng_NPI"] + list(_b_provider_cols))
+ .unique(subset="Rndrng_NPI", keep="first", maintain_order=True)
+ .sort("Rndrng_NPI").rename(_b_provider_cols))
+ svc = (df.select(["HCPCS_Cd", "HCPCS_Desc", "HCPCS_Drug_Ind"])
+ .unique(subset="HCPCS_Cd", keep="first", maintain_order=True)
+ .sort("HCPCS_Cd").with_row_index("d"))
+ n_p = len(prov)
+ df = (df.join(prov.select("Rndrng_NPI").with_row_index("p"), on="Rndrng_NPI")
+ .join(svc.select("HCPCS_Cd", "d"), on="HCPCS_Cd"))
+ g = Graph(directed=False)
+ g.add_vertex(n_p + len(svc))
+ g.vp.is_provider = g.new_vp("bool", vals=numpy.arange(g.num_vertices()) < n_p)
+ g.vp.npi = g.new_vp("int64_t", vals=numpy.concatenate(
+ (prov["Rndrng_NPI"].to_numpy(), numpy.zeros(len(svc), dtype="int64"))))
+ g.vp.name = g.new_vp("string", vals=prov["name"].to_list() + svc["HCPCS_Cd"].to_list())
+ for c in list(_b_provider_cols.values())[1:]:
+ g.vp[c] = g.new_vp("string", vals=prov[c].to_list() + [""] * len(svc))
+ g.vp.hcpcs_code = g.new_vp("string", vals=[""] * n_p + svc["HCPCS_Cd"].to_list())
+ g.vp.description = g.new_vp("string", vals=[""] * n_p + svc["HCPCS_Desc"].to_list())
+ g.vp.is_drug = g.new_vp("bool", vals=[False] * n_p +
+ [x == "Y" for x in svc["HCPCS_Drug_Ind"].to_list()])
+ del prov, svc
+ g.add_edge_list(numpy.column_stack((df["p"].to_numpy(),
+ df["d"].to_numpy().astype("int64") + n_p)))
+ for c, k in _b_edge_cols.items():
+ if k == "place_of_service":
+ g.ep[k] = g.new_ep("string", vals=df[c].to_list())
+ elif k in ("beneficiaries", "services", "beneficiary_days"):
+ g.ep[k] = g.new_ep("double", vals=df[c].cast(pl.Float64).to_numpy())
+ else:
+ g.ep[k] = g.new_ep("double", vals=df[c].cast(pl.Float64).to_numpy())
+ return g
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+ import polars as pl
+ name = __name__.split(".")[-1]
+ for fname, alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ if alt.startswith("part_b_"):
+ yield alt, _parse_part_b(name, fname)
+ continue
+ with open_upstream_file(name, fname, "rb") as f:
+ df = pl.read_csv(f, n_rows=_check_rows, columns=["Prscrbr_NPI", "Brnd_Name", "Gnrc_Name"] +
+ list(_prescriber_cols) + list(_edge_cols),
+ schema_overrides={c: pl.String for c in
+ ["Brnd_Name", "Gnrc_Name"] + list(_prescriber_cols)} |
+ {"Prscrbr_NPI": pl.Int64,
+ "Tot_Clms": pl.Float64,
+ "Tot_30day_Fills": pl.Float64,
+ "Tot_Day_Suply": pl.Float64,
+ "Tot_Drug_Cst": pl.Float64,
+ "Tot_Benes": pl.Float64},
+ infer_schema=False)
+ df = df.with_columns(pl.col(c).fill_null("") for c in
+ ["Brnd_Name", "Gnrc_Name"] + list(_prescriber_cols))
+
+ # the prescribers, in the order of their NPIs (the attributes of
+ # the first row of each)
+ pres = (df.select(["Prscrbr_NPI"] + list(_prescriber_cols))
+ .unique(subset="Prscrbr_NPI", keep="first", maintain_order=True)
+ .sort("Prscrbr_NPI")
+ .rename(_prescriber_cols))
+ # the drugs, in the order of their names
+ drugs = (df.select(["Brnd_Name", "Gnrc_Name"]).unique()
+ .sort(["Brnd_Name", "Gnrc_Name"])
+ .with_row_index("d"))
+ n_p = len(pres)
+ df = (df.join(pres.select("Prscrbr_NPI").with_row_index("p"),
+ on="Prscrbr_NPI")
+ .join(drugs, on=["Brnd_Name", "Gnrc_Name"]))
+
+ g = Graph(directed=False)
+ g.add_vertex(n_p + len(drugs))
+ g.vp.is_prescriber = g.new_vp("bool", vals=numpy.arange(g.num_vertices()) < n_p)
+ g.vp.npi = g.new_vp("int64_t", vals=numpy.concatenate(
+ (pres["Prscrbr_NPI"].to_numpy(), numpy.zeros(len(drugs), dtype="int64"))))
+ g.vp.name = g.new_vp("string", vals=pres["name"].to_list() + drugs["Brnd_Name"].to_list())
+ for c in ["first_name", "city", "state", "specialty", "specialty_source"]:
+ g.vp[c] = g.new_vp("string", vals=pres[c].to_list() + [""] * len(drugs))
+ g.vp.generic_name = g.new_vp("string", vals=[""] * n_p + drugs["Gnrc_Name"].to_list())
+ del pres, drugs
+ g.add_edge_list(numpy.column_stack((df["p"].to_numpy(),
+ df["d"].to_numpy().astype("int64") + n_p)))
+ for c, k in _edge_cols.items():
+ x = df[c]
+ if k == "beneficiaries":
+ x = x.fill_null(-1)
+ if k in ("claims", "days_supply", "beneficiaries"):
+ g.ep[k] = g.new_ep("int64_t", vals=x.cast(pl.Int64).to_numpy())
+ else:
+ g.ep[k] = g.new_ep("double", vals=x.to_numpy())
+ del df
+ yield alt, g