Download intermediate_leaderboard.py from openadmet/OpenADMET-ExpansionRx-Challenge: direct link, hf CLI and curl.
- Browser
- Download file 5.65 kB
-
https://huggingface.co/spaces/openadmet/OpenADMET-ExpansionRx-Challenge/resolve/main/intermediate_leaderboard.py
- Command line
-
hf download hf://spaces/openadmet/OpenADMET-ExpansionRx-Challenge/intermediate_leaderboard.py
-
curl -L -o intermediate_leaderboard.py https://huggingface.co/spaces/openadmet/OpenADMET-ExpansionRx-Challenge/resolve/main/intermediate_leaderboard.py
5.65 kB
| from statsmodels.stats.multicomp import pairwise_tukeyhsd | |
| from cld import cld | |
| from utils import ( | |
| check_page_exists, | |
| map_metric_to_stats, | |
| fetch_dataset_df, | |
| ) | |
| from about import ENDPOINTS, LB_COLS, results_repo_test, METRICS | |
| import pandas as pd | |
| def validate_hf_username(username): | |
| username = str(username).strip() | |
| hf_url = f"https://huggingface.co/{username}" | |
| return check_page_exists(hf_url, delay=1, max_retries=10) | |
| # return True # For testing purposes, assume all usernames are valid | |
| def validate_model_details(tag): | |
| if tag is None: | |
| return "Not submitted" | |
| safe_tag = str(tag).strip() | |
| if not safe_tag.startswith("https://"): | |
| return "Invalid link" | |
| is_real_url = check_page_exists(safe_tag, delay=2) | |
| if not is_real_url: | |
| return "Invalid link" | |
| else: | |
| return safe_tag | |
| def make_intermediate_lb(): | |
| df_latest, df_latest_raw = fetch_dataset_df( | |
| download_raw=True, | |
| test_repo=results_repo_test | |
| ) | |
| # Make all usernames lowercase | |
| df_latest_raw["hf_username"] = df_latest_raw["hf_username"].str.lower() | |
| # HF username validation | |
| hf_usernames = df_latest_raw["hf_username"].unique() | |
| valid_hf_usernames = {username: validate_hf_username(username) for username in hf_usernames} | |
| # print all users and their validation status | |
| for username, is_valid in valid_hf_usernames.items(): | |
| print(f"Username: {username}, Valid: {is_valid}") | |
| df_latest_raw["hf_user_valid"] = df_latest_raw["hf_username"].map(valid_hf_usernames) | |
| # drop invalid usernames | |
| df_latest_raw = df_latest_raw[df_latest_raw["hf_user_valid"]].reset_index(drop=True) | |
| # make sure to only keep the latest submission per user for the 'Average' endpoint | |
| df_latest_raw["submission_time"] = pd.to_datetime(df_latest_raw["submission_time"]) | |
| df_latest_raw = df_latest_raw.query("Endpoint == 'Average'") | |
| df_latest_raw['latest_time_per_user'] = df_latest_raw.groupby('hf_username')['submission_time'].transform('max') | |
| latest_submissions_df = df_latest_raw[df_latest_raw['submission_time'] == df_latest_raw['latest_time_per_user']].copy() | |
| # Fix to order by the mean RAE and not the RAE of all samples (slight missmatch for some users) | |
| latest_submissions_df['mean_RAE'] = latest_submissions_df.groupby('hf_username')['RAE'].transform('mean') | |
| latest_submissions_df = latest_submissions_df.sort_values( | |
| by=['mean_RAE', 'Sample'], ascending=True | |
| ).reset_index(drop=True) | |
| # Get the unique users in the order of their first appearance | |
| unique_users_ordered = latest_submissions_df['user'].unique() | |
| # Create a mapping dictionary: original_user -> prefixed_user | |
| user_mapping = {} | |
| for idx, user in enumerate(unique_users_ordered): | |
| # The prefix is the index starting from 0, formatted to be 3 digits (001, 002, etc.) | |
| # We use idx + 1 to start the sequence from 001 instead of 000 | |
| prefix = f"{idx + 1:03d}" | |
| prefixed_user = f"{prefix}___{user}" | |
| user_mapping[user] = prefixed_user | |
| # Apply the mapping to create a new column with prefixed usernames | |
| latest_submissions_df['user'] = latest_submissions_df['user'].map(user_mapping) | |
| # Perform Tukey's HSD test | |
| tukey = pairwise_tukeyhsd(endog=latest_submissions_df['RAE'], groups=latest_submissions_df['user'], alpha=0.05) | |
| tukey_df = pd.DataFrame(data=tukey._results_table.data[1:], | |
| columns=tukey._results_table.data[0]) | |
| # add CLDs | |
| cld_dict = cld(tukey_df) | |
| cld_df = pd.DataFrame(cld_dict.items(),columns=["group","letter"]).sort_values("group") | |
| cld_df.letter = [",".join(x) for x in cld_df.letter] | |
| cld_df["user"] = cld_df.group | |
| # clean up CLD letters for extended alphabet (i.e with @ symbols) | |
| def clean_up(ser): | |
| ser = ser.split(",") | |
| # rejoin for late in alphabet | |
| if "@" in ser and len(ser) == 2: | |
| let = "@" + ser[1] | |
| elif "@" in ser and len(ser) == 4: | |
| let = "@" + ser[2] + "," + "@" + ser[3] | |
| else: | |
| let = ",".join(ser) | |
| return let | |
| cld_df["fixed_letter"] = cld_df["letter"].apply(lambda x: clean_up(x)) | |
| report_cols = latest_submissions_df[['user', 'model_report']].drop_duplicates(keep='first') | |
| # gather means and stds for each metric for each user | |
| for metric in METRICS: | |
| metric_stats = latest_submissions_df.groupby('user')[metric].agg(['mean', 'std']).reset_index() | |
| metric_stats = metric_stats.rename(columns={'mean': f'{metric}_mean', 'std': f'{metric}_std'}) | |
| metric_stats[f"{metric}_display"] = metric_stats.apply( | |
| lambda row: f"{row[f'{metric}_mean']:.4f} ± {row[f'{metric}_std']:.4f}", axis=1 | |
| ) | |
| cld_df = metric_stats[['user', f'{metric}_mean', f'{metric}_std', f'{metric}_display']].merge(cld_df, on='user', how='left') | |
| # re-sort by RAE mean, lowest is best | |
| cld_df = cld_df.sort_values(by='RAE_mean', ascending=True).reset_index(drop=True) | |
| cld_df = cld_df.merge(report_cols, on='user', how='inner') | |
| cld_df['user'] = cld_df['user'].str.split('___').str[1] | |
| cld_subset = cld_df[['user', 'fixed_letter'] + [f'{metric}_display' for metric in METRICS]+ ['model_report']] | |
| cld_subset = cld_subset.rename(columns={'user': 'user', 'fixed_letter': 'CLD', 'model_report': 'model details'}) | |
| cld_subset['model details'] = cld_subset['model details'].apply(lambda x: validate_model_details(x)).astype(str) | |
| print(cld_subset.head()) | |
| cld_subset.to_csv("leaderboard_cld_results.csv", index=False) | |
| if __name__ == "__main__": | |
| make_intermediate_lb() |