Skip to content

Generate Predicted Agency Stop-Level Ridership Data #2137

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
shweta487 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Generate Predicted Agency Stop-Level Ridership Data #2137

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
280 changes: 231 additions & 49 deletions ahsc_grant/ahsc_grant_cycle2/02_Stop_Route_Data_Cleaning.ipynb
View file
Open in desktop

Large diffs are not rendered by default.

729 changes: 440 additions & 289 deletions ahsc_grant/ahsc_grant_cycle2/03_Stops_Ridership_Data_weekday.ipynb
View file
Open in desktop

Large diffs are not rendered by default.

535 changes: 535 additions & 0 deletions ahsc_grant/ahsc_grant_cycle2/04a_prepare_acs_block.ipynb
View file
Open in desktop

Large diffs are not rendered by default.

395 changes: 395 additions & 0 deletions ahsc_grant/ahsc_grant_cycle2/05a_process_lehd_block.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,395 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "8dffcb05-ee42-42a8-9260-6ff90841ce75",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"os.environ[\"CALITP_BQ_MAX_BYTES\"] = str(800_000_000_000)\n",
"\n",
"import shared_utils\n",
"import pandas as pd\n",
"import geopandas as gpd\n",
"\n",
"import gcsfs\n",
"from calitp_data_analysis import get_fs\n",
"from calitp_data_analysis import geography_utils, utils\n",
"fs = get_fs()\n",
"import re\n",
"import google.auth\n",
"import os\n",
"import gcsfs\n",
"credentials, project = google.auth.default()\n",
"fs = gcsfs.GCSFileSystem()"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "38d86912-abd1-44fc-9231-3d798ab63923",
"metadata": {},
"outputs": [],
"source": [
"GCS_FILE_PATH = 'gs://calitp-analytics-data/data-analyses/ahsc_grant/ahsc_riderships/AHSC_2026'"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "ec3e92b4-3cf7-42d1-b194-1ea9df797068",
"metadata": {},
"outputs": [],
"source": [
"url_tot = \"https://lehd.ces.census.gov/data/lodes/LODES8/ca/wac/ca_wac_S000_JT00_2023.csv.gz\"\n",
"url_prv = \"https://lehd.ces.census.gov/data/lodes/LODES8/ca/wac/ca_wac_S000_JT01_2023.csv.gz\"\n"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "20205a3f-cc20-42e3-b66d-3021fc947735",
"metadata": {},
"outputs": [],
"source": [
"lodes_tot = pd.read_csv(url_tot, compression='gzip')\n",
"lodes_prv = pd.read_csv(url_prv, compression='gzip')"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "352ceb54-3721-48de-bcaa-c2ae63e0a467",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>w_geocode</th>\n",
" <th>C000</th>\n",
" <th>CA01</th>\n",
" <th>CA02</th>\n",
" <th>CA03</th>\n",
" <th>CE01</th>\n",
" <th>CE02</th>\n",
" <th>CE03</th>\n",
" <th>CNS01</th>\n",
" <th>CNS02</th>\n",
" <th>...</th>\n",
" <th>CFA02</th>\n",
" <th>CFA03</th>\n",
" <th>CFA04</th>\n",
" <th>CFA05</th>\n",
" <th>CFS01</th>\n",
" <th>CFS02</th>\n",
" <th>CFS03</th>\n",
" <th>CFS04</th>\n",
" <th>CFS05</th>\n",
" <th>createdate</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>60014001001003</td>\n",
" <td>21</td>\n",
" <td>2</td>\n",
" <td>10</td>\n",
" <td>9</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>21</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>...</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>20251202</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>60014001001010</td>\n",
" <td>3</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>3</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>...</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>20251202</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>60014001001011</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>...</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>20251202</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>60014001001013</td>\n",
" <td>3</td>\n",
" <td>0</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>3</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>...</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>20251202</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>60014001001015</td>\n",
" <td>3</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>...</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>20251202</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>5 rows × 53 columns</p>\n",
"</div>"
],
"text/plain": [
" w_geocode C000 CA01 CA02 CA03 CE01 CE02 CE03 CNS01 CNS02 \\\n",
"0 60014001001003 21 2 10 9 0 0 21 0 0 \n",
"1 60014001001010 3 0 1 2 0 0 3 0 0 \n",
"2 60014001001011 1 0 0 1 0 1 0 0 0 \n",
"3 60014001001013 3 0 2 1 0 0 3 0 0 \n",
"4 60014001001015 3 0 1 2 1 1 1 0 0 \n",
"\n",
" ... CFA02 CFA03 CFA04 CFA05 CFS01 CFS02 CFS03 CFS04 CFS05 \\\n",
"0 ... 0 0 0 0 0 0 0 0 0 \n",
"1 ... 0 0 0 0 0 0 0 0 0 \n",
"2 ... 0 0 0 0 0 0 0 0 0 \n",
"3 ... 0 0 0 0 0 0 0 0 0 \n",
"4 ... 0 0 0 0 0 0 0 0 0 \n",
"\n",
" createdate \n",
"0 20251202 \n",
"1 20251202 \n",
"2 20251202 \n",
"3 20251202 \n",
"4 20251202 \n",
"\n",
"[5 rows x 53 columns]"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"lodes_tot.head(5)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "05e4f575-9898-43ae-9078-ab5f69328fea",
"metadata": {},
"outputs": [],
"source": [
"\n",
"\n",
"for df in [lodes_tot, lodes_prv]:\n",
" df['GEOID'] = df['w_geocode'].astype(str).str.zfill(15).str[:12]\n",
"\n",
"jobs_bg = lodes_tot.groupby('GEOID').agg(jobs_tot=('C000', 'sum')).reset_index()\n",
"jobs_prv = lodes_prv.groupby('GEOID').agg(jobs_prv=('C000', 'sum')).reset_index()\n",
"\n",
"jobs_bg = jobs_bg.merge(jobs_prv, on='GEOID', how='left')\n",
"jobs_bg.rename(columns={'GEOID': 'h_geocode'}, inplace=True)\n",
"jobs_bg['segment'] = 'S000'\n",
"jobs_bg['year'] = 2023\n",
"jobs_bg = jobs_bg[['h_geocode', 'segment', 'year', 'jobs_tot', 'jobs_prv']]"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "2985a9b2-c3c9-45ba-b870-e22b28c7ce4b",
"metadata": {},
"outputs": [],
"source": [
"#Sort by GEOID\n",
"jobs_bg = jobs_bg.sort_values(by='h_geocode')\n",
"\n",
"#Create jobs_fed = jobs_total - jobs_prv\n",
"jobs_bg['jobs_fed'] = jobs_bg['jobs_tot'] - jobs_bg['jobs_prv']\n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "7e5fd53f-bed2-4da3-ac7a-3da6e72b2958",
"metadata": {},
"outputs": [],
"source": [
"jobs_bg['GEOID'] = jobs_bg['h_geocode'].astype(str).str[:12]"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "e54d0a19-4cde-4f0c-a6b9-fb1a1a31a50a",
"metadata": {},
"outputs": [],
"source": [
"grouped_df = jobs_bg.groupby('GEOID', as_index=False).agg({\n",
" 'jobs_tot': 'sum',\n",
" 'jobs_prv': 'sum',\n",
" 'jobs_fed': 'sum',\n",
" 'year': 'first'\n",
"})"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "b7e75874-82f2-4151-a065-4c6cbe3979b4",
"metadata": {},
"outputs": [],
"source": [
"def export_gdf(gdf, filename: str):\n",
" \n",
" gdf.to_parquet(f\"{filename}.parquet\")\n",
" \n",
" fs.put(\n",
" f\"{filename}.parquet\",\n",
" f\"{GCS_FILE_PATH}/{filename}.parquet\",\n",
" token = credentials.token\n",
" )\n",
" \n",
" os.remove(f\"{filename}.parquet\")\n",
" print(f\"saved {GCS_FILE_PATH}/{filename}.parquet\")\n",
" \n",
" return"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "18bc2764-9338-42c8-a0a4-c04523bb70bb",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"saved gs://calitp-analytics-data/data-analyses/ahsc_grant/ahsc_riderships/AHSC_2026/job_density_block_2023.parquet\n"
]
}
],
"source": [
"export_gdf(grouped_df, \"job_density_block_2023\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Pyproject Local (use-venv)",
"language": "python",
"name": "pyproject_local_kernel_use_venv"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Loading
Loading