From c54e25a979a51d043c560778c8a45c2d5f160ccb Mon Sep 17 00:00:00 2001 From: crockettz Date: Wed, 15 Jul 2026 11:37:46 -0400 Subject: [PATCH] Modifying the user stats to do smaller batches and more retries; modifying the slack notifications to ignore the UPS errors. --- bin/crontab_scripts/notify.sh | 5 ++++- source/daily_cron_jobs/upload_user_stats.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/crontab_scripts/notify.sh b/bin/crontab_scripts/notify.sh index b674177..2a1fca3 100644 --- a/bin/crontab_scripts/notify.sh +++ b/bin/crontab_scripts/notify.sh @@ -14,11 +14,14 @@ FILES=$(find "$LOG_DIR" -type f \( -name "*_$TODAY.log" -o -name "*_$YESTERDAY.l # Initialize an empty array to store files with matches MATCHED_FILES=() +# Ignore errors with user profile service that trigger a re-try +IGNORE_PATTERN="SKIPPING profile batch for users" + # Check if any files were found if [ -n "$FILES" ]; then # Search for "exception" or "traceback" or "Errno" in the found files for FILE in $FILES; do - if grep -qiE "exception|traceback|Errno" "$FILE"; then + if grep -iE "exception|traceback|Errno" "$FILE" | grep -qivE "$IGNORE_PATTERN"; then MATCHED_FILES+=("$FILE") fi done diff --git a/source/daily_cron_jobs/upload_user_stats.py b/source/daily_cron_jobs/upload_user_stats.py index 2f6032c..f4381e4 100644 --- a/source/daily_cron_jobs/upload_user_stats.py +++ b/source/daily_cron_jobs/upload_user_stats.py @@ -13,7 +13,7 @@ user_stats_dict = methods_upload_user_stats.get_user_narrative_stats(user_stats_dict) user_stats_dict = methods_upload_user_stats.get_user_orcidlinks(user_stats_dict) #user_stats_dict = methods_upload_user_stats.get_institution_and_country(user_stats_dict) -user_stats_dict = methods_upload_user_stats.get_profile_info(user_stats_dict, batch_size=500, max_retries=1) +user_stats_dict = methods_upload_user_stats.get_profile_info(user_stats_dict, batch_size=250, max_retries=3) print("--- gather data %s seconds ---" % (time.time() - start_time)) methods_upload_user_stats.upload_user_data(user_stats_dict) print(