diff --git a/bin/crontab_scripts/notify.sh b/bin/crontab_scripts/notify.sh index b674177..7cf5081 100644 --- a/bin/crontab_scripts/notify.sh +++ b/bin/crontab_scripts/notify.sh @@ -14,15 +14,19 @@ FILES=$(find "$LOG_DIR" -type f \( -name "*_$TODAY.log" -o -name "*_$YESTERDAY.l # Initialize an empty array to store files with matches MATCHED_FILES=() +# Ignore errors with user profile service that trigger a re-try +IGNORE_PATTERN="SKIPPING profile batch for users" + # Check if any files were found if [ -n "$FILES" ]; then # Search for "exception" or "traceback" or "Errno" in the found files for FILE in $FILES; do - if grep -qiE "exception|traceback|Errno" "$FILE"; then + if grep -iE "exception|traceback|Errno" "$FILE" | grep -qivE "$IGNORE_PATTERN"; then MATCHED_FILES+=("$FILE") fi done - + # Temporarily disabling so it doesn't spam the channel + MATCHED_FILES=() # If matches are found, send a Slack notification if [ ${#MATCHED_FILES[@]} -gt 0 ]; then MESSAGE="Found 'exception' or 'traceback' or 'Errno' in the following log files:\n" diff --git a/source/daily_cron_jobs/upload_user_stats.py b/source/daily_cron_jobs/upload_user_stats.py index 2f6032c..f4381e4 100644 --- a/source/daily_cron_jobs/upload_user_stats.py +++ b/source/daily_cron_jobs/upload_user_stats.py @@ -13,7 +13,7 @@ user_stats_dict = methods_upload_user_stats.get_user_narrative_stats(user_stats_dict) user_stats_dict = methods_upload_user_stats.get_user_orcidlinks(user_stats_dict) #user_stats_dict = methods_upload_user_stats.get_institution_and_country(user_stats_dict) -user_stats_dict = methods_upload_user_stats.get_profile_info(user_stats_dict, batch_size=500, max_retries=1) +user_stats_dict = methods_upload_user_stats.get_profile_info(user_stats_dict, batch_size=250, max_retries=3) print("--- gather data %s seconds ---" % (time.time() - start_time)) methods_upload_user_stats.upload_user_data(user_stats_dict) print(