""" ============= 5. SHAP Plots ============= This file shows interpretation of LGBMClassifier model using SHAP. """ import pandas as pd import numpy as np import matplotlib.pyplot as plt from shap import TreeExplainer from shap import summary_plot from utils import SAVE from utils import CATEGORIES from utils import LABEL_MAP from utils import version_info from utils import set_rcParams from utils import make_data, return_train_test from utils import get_no_gene_model from utils import bar_pie from utils import shap_scatter_plots # %% set_rcParams() # %% version_info() # %% def make_classes(exp): colors = {'Physicochemical': '#006db6', 'ARGs': '#f8aa59', 'Antibiotics': '#b72c10' } classes = [] colors_ = [] for f in exp.feature_names: if f in CATEGORIES['Physicochemical']: classes.append('Physicochemical') colors_.append(colors['Physicochemical']) elif f in CATEGORIES['ARGs']: classes.append('ARGs') colors_.append(colors['ARGs']) elif f in CATEGORIES['Antibiotics']: classes.append('Antibiotics') colors_.append(colors['Antibiotics']) else: raise ValueError return classes, colors_ # %% # MCR-1 # ===== target = 'MCR-1' TrainX, TrainY, TestX, TestY, inputs, encoders = return_train_test(target, 'no_genes', return_encoders=True) model = get_no_gene_model(target, True, 'LGBMClassifier') cols = TrainX.columns.tolist() TrainX = pd.DataFrame(model._transform_x(TrainX), columns=cols) # Create a SHAP explainer explainer = TreeExplainer(model._model, TrainX, feature_names=TrainX.columns) # Calculate the SHAP values for the test set shap_values = explainer.shap_values(TestX, TestY) # %% # summary plot # ------------ summary_plot(shap_values, TestX, max_display=34, feature_names=[LABEL_MAP[n] if n in LABEL_MAP else n for n in TrainX.columns], show=False, class_names=['yes', 'no']) if SAVE: plt.savefig(f"results/figures/shap_summary_{target}.png", dpi=600, bbox_inches="tight") plt.tight_layout() plt.show() # %% # bar chart # --------- sv_bar = np.mean(np.abs(shap_values), axis=0) classes, colors_ = make_classes(explainer) df_with_classes = pd.DataFrame( {'features': explainer.feature_names, 'classes': classes, 'mean_shap': sv_bar, 'colors': colors_ }) print(df_with_classes) bar_pie(df_with_classes, save=SAVE, name=f"{target}", show=True) # %% # scatter plot # ------------ feature = 'season' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'T (℃)' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'pH' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'TDS (mg/L)' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'DO (mg/L)' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'TN (mg/L)' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% # OXA48 # ===== target = 'OXA48' TrainX, TrainY, TestX, TestY, inputs = return_train_test(target, 'no_genes') model = get_no_gene_model(target, True, 'LGBMClassifier') cols = TrainX.columns.tolist() TrainX = pd.DataFrame(model._transform_x(TrainX), columns=cols) #TestX = pd.DataFrame(model._transform_x(TestX), columns=cols) # Create a SHAP explainer explainer = TreeExplainer(model._model, TrainX, feature_names=TrainX.columns) # Calculate the SHAP values for the test set shap_values = explainer.shap_values(TestX, TestY) # %% # summary plot # ------------ summary_plot(shap_values, TestX, max_display=34, feature_names=[LABEL_MAP[n] if n in LABEL_MAP else n for n in TrainX.columns], show=False, class_names=['yes', 'no']) if SAVE: plt.savefig(f"results/figures/shap_summary_{target}.png", dpi=600, bbox_inches="tight") plt.tight_layout() plt.show() # %% # bar chart # --------- sv_bar = np.mean(np.abs(shap_values), axis=0) classes, colors_ = make_classes(explainer) df_with_classes = pd.DataFrame( {'features': explainer.feature_names, 'classes': classes, 'mean_shap': sv_bar, 'colors': colors_ }) print(df_with_classes) bar_pie(df_with_classes, save=SAVE, name=f"{target}", show=True) # %% # scatter plot # ------------ feature = 'season' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'T (℃)' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'pH' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'TDS (mg/L)' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'DO (mg/L)' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'TN (mg/L)' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% # TEM # === target = 'TEM' TrainX, TrainY, TestX, TestY, inputs = return_train_test(target, 'no_genes') model = get_no_gene_model(target, True, 'LGBMClassifier') cols = TrainX.columns.tolist() TrainX = pd.DataFrame(model._transform_x(TrainX), columns=cols) #TestX = pd.DataFrame(model._transform_x(TestX), columns=cols) # Create a SHAP explainer explainer = TreeExplainer(model._model, TrainX, feature_names=TrainX.columns) # Calculate the SHAP values for the test set shap_values = explainer.shap_values(TestX, TestY) # %% # summary plot # ------------ summary_plot(shap_values, TestX, max_display=34, feature_names=[LABEL_MAP[n] if n in LABEL_MAP else n for n in TrainX.columns], show=False, class_names=['yes', 'no']) if SAVE: plt.savefig(f"results/figures/shap_summary_{target}.png", dpi=600, bbox_inches="tight") plt.tight_layout() plt.show() # %% # bar chart # --------- sv_bar = np.mean(np.abs(shap_values), axis=0) classes, colors_ = make_classes(explainer) df_with_classes = pd.DataFrame( {'features': explainer.feature_names, 'classes': classes, 'mean_shap': sv_bar, 'colors': colors_ }) print(df_with_classes) bar_pie(df_with_classes, save=SAVE, name=f"{target}", show=True) # %% # scatter plot # ------------ feature = 'season' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'T (℃)' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'pH' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'TDS (mg/L)' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'DO (mg/L)' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'TN (mg/L)' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% # CTX-M # ===== target = 'CTX-M' TrainX, TrainY, TestX, TestY, inputs = return_train_test(target, 'no_genes') model = get_no_gene_model(target, 'LGBMClassifier') cols = TrainX.columns.tolist() #TrainX = pd.DataFrame(model._transform_x(TrainX), columns=cols) #TestX = pd.DataFrame(model._transform_x(TestX), columns=cols) # Create a SHAP explainer explainer = TreeExplainer(model._model, TrainX, feature_names=TrainX.columns) # Calculate the SHAP values for the test set shap_values = explainer.shap_values(TestX, TestY, check_additivity=False) # %% # summary plot # ------------ summary_plot(shap_values, TestX, max_display=34, feature_names=[LABEL_MAP[n] if n in LABEL_MAP else n for n in TrainX.columns], show=False, class_names=['yes', 'no']) if SAVE: plt.savefig(f"results/figures/shap_summary_{target}.png", dpi=600, bbox_inches="tight") plt.tight_layout() plt.show() # %% # bar chart # --------- sv_bar = np.mean(np.abs(shap_values), axis=0) classes, colors_ = make_classes(explainer) df_with_classes = pd.DataFrame( {'features': explainer.feature_names, 'classes': classes, 'mean_shap': sv_bar, 'colors': colors_ }) print(df_with_classes) bar_pie(df_with_classes, save=SAVE, name=f"{target}", show=True) # %% # scatter plot # ------------ feature = 'season' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'T (℃)' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'pH' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'TDS (mg/L)' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'DO (mg/L)' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target) # %% feature = 'TN (mg/L)' if feature in TrainX: shap_scatter_plots( shap_values, TestX, feature_name=feature, encoders=encoders, save=SAVE, name=target)