DenseNet121_Chexpert_CWBCE_E9_B32_C0_N12_D320_Masked_NoFinding - TobiasSchmidtDE/DeepL-MedicalImaging GitHub Wiki
Version: 1
Trained DenseNet121 architecture using the 'Chexpert_CWBCE_E9_B32_C0_N12_D320_Masked_NoFinding' benchmark. The benchmark was initialized for the chexpert_full dataset with batch size of 32, shuffle set to True and images rescaled to dimension (320, 320). The training was done for 9 epochs using the Adam optimizer and weighted_binary_crossentropy loss. A total of 12 labels/pathologies were included in the training and encoded using the 'uzeroes' method. The traing set included 142320 number of sample, the validation set 36162, and the test set 44932.
from pathlib import Path
from dotenv import load_dotenv, find_dotenv
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.pyplot import figure
import json
import os
import re
import pprint
basepath = Path(os.getcwd())
if basepath.name != "idp-radio-1":
os.chdir(basepath.parent.parent)
print(os.getcwd())
load_dotenv(find_dotenv())
from src.preprocessing.split.train_test_split import train_test_split/srv/idp-radio-1
data = json.loads(os.environ['EXP_DATA'])
history = data['history']for s in data["description"].split(".")[:-1]:
print(s + ".\n")Trained DenseNet121 architecture using the 'Chexpert_CWBCE_E9_B32_C0_N12_D320_Masked_NoFinding' benchmark.
The benchmark was initialized for the chexpert_full dataset with batch size of 32, shuffle set to True and images rescaled to dimension (320, 320).
The training was done for 9 epochs using the Adam optimizer and weighted_binary_crossentropy loss.
A total of 12 labels/pathologies were included in the training and encoded using the 'uzeroes' method.
The traing set included 142320 number of sample, the validation set 36162, and the test set 44932.
# if there are any metrics that were renamed, add this new name here as ("default_name":"new_name")
metric_custom_names={"auc":"AUC_ROC"}
metric_names = [re.sub("([a-z0-9])([A-Z])","\g<1> \g<2>",name) for name in data["benchmark"]["metrics"]]
metric_keys = [re.sub("([a-z0-9])([A-Z])","\g<1>_\g<2>",name).lower() for name in data["benchmark"]["metrics"]]
for default_name, custom_name in metric_custom_names.items():
if not default_name in history.keys() and default_name in metric_keys:
#replace default name with custom name
metric_keys[metric_keys.index(default_name)]=custom_namedef print_or_plot_metric(metric_key, metric_name, figure_name):
if len(history[metric_key]) == 1:
print("Data for {m_name} only available for a single epoch. \nSkipping plot and printing data...".format(m_name=metric_name))
print('Train {}: '.format(metric_name), history[metric_key])
print('Validation {}: '.format(metric_name), history['val_'+metric_key])
print()
else:
plot_epoch_metric(metric_key, metric_name, figure_name)
def plot_epoch_metric(metric_key, metric_name, figure_name):
figure(num=None, figsize=(10, 6))
plt.plot(history[metric_key])
if 'val_'+metric_key in history.keys():
plt.plot(history['val_'+metric_key])
plt.title(figure_name)
plt.ylabel(metric_name)
plt.xlabel('Epoch')
if 'val_'+metric_key in history.keys():
plt.legend(['Train', 'Validation'], loc='upper left')
plt.show()
for i, metric_key in enumerate(metric_keys):
print_or_plot_metric(metric_key, metric_names[i], "Model "+metric_names[i])
print_or_plot_metric("loss", "Loss", "Model loss")
if "lr" in history.keys():
plot_epoch_metric("lr", "Learning Rate", "Learning Rate")
if 'classification_report' in data.keys() and data['classification_report']:
print(data['classification_report']) precision recall f1-score support
Enlarged Cardiomediastinum 0.05 0.61 0.09 2214
Cardiomegaly 0.12 0.45 0.19 5294
Lung Opacity 0.47 0.63 0.54 21324
Lung Lesion 0.04 0.64 0.08 1901
Edema 0.23 0.46 0.31 10461
Consolidation 0.07 0.51 0.12 3063
Pneumonia 0.03 0.52 0.05 1225
Atelectasis 0.15 0.66 0.25 6912
Pneumothorax 0.09 0.24 0.13 3894
Pleural Effusion 0.39 0.51 0.44 17656
Pleural Other 0.02 0.61 0.03 747
Fracture 0.04 0.74 0.08 1863
micro avg 0.14 0.55 0.22 76554
macro avg 0.14 0.55 0.19 76554
weighted avg 0.29 0.55 0.35 76554
samples avg 0.11 0.45 0.16 76554
if 'test' in data.keys() and data['test']:
for score_name, score in data["test"].items():
print('Test {}: '.format(score_name), score)
Test loss: 0.6720725893974304
Test auc: 0.20295846462249756
Test precision: 0.7850145101547241
Test recall: 0.49887436628341675
Test f2_score: 0.5317652225494385
Test binary_accuracy: 0.410776287317276
Test accuracy_enlarged_cardiomediastinum: 0.6298406720161438
Test accuracy_cardiomegaly: 0.6489806771278381
Test accuracy_lung_opacity: 0.35702839493751526
Test accuracy_lung_lesion: 0.6740185022354126
Test accuracy_edema: 0.5420858263969421
Test accuracy_consolidation: 0.4857339859008789
Test accuracy_pneumonia: 0.4266001880168915
Test accuracy_atelectasis: 0.7850752472877502
Test accuracy_pneumothorax: 0.757210910320282
Test accuracy_pleural_effusion: 0.38796404004096985
Test accuracy_pleural_other: 0.2759280800819397
Test accuracy_fracture: 0.5612546801567078
Test auc_enlarged_cardiomediastinum: 0.7878790497779846
Test auc_cardiomegaly: 0.7057811617851257
Test auc_lung_opacity: 0.5523748993873596
Test auc_lung_lesion: 0.786817729473114
Test auc_edema: 0.7050762176513672
Test auc_consolidation: 0.6472144722938538
Test auc_pneumonia: 0.6149752736091614
Test auc_atelectasis: 0.7703368663787842
Test auc_pneumothorax: 0.8488591313362122
Test auc_pleural_effusion: 0.6184005737304688
Test auc_pleural_other: 0.4659003019332886
Test auc_fracture: 0.055088941007852554
Test precision_enlarged_cardiomediastinum: 0.21575410664081573
Test precision_cardiomegaly: 0.5976707935333252
Test precision_lung_opacity: 0.04310140386223793
Test precision_lung_lesion: 0.3996259272098541
Test precision_edema: 0.10505187511444092
Test precision_consolidation: 0.036201778799295425
Test precision_pneumonia: 0.18209105730056763
Test precision_atelectasis: 0.22370313107967377
Test precision_pneumothorax: 0.6458684802055359
Test precision_pleural_effusion: 0.01964304968714714
Test precision_pleural_other: 0.03904535621404648
Test precision_fracture: 0.6784101128578186
Test recall_enlarged_cardiomediastinum: 0.8128069639205933
Test recall_cardiomegaly: 0.7966141700744629
Test recall_lung_opacity: 0.6696475744247437
Test recall_lung_lesion: 0.7965777516365051
Test recall_edema: 0.7603656649589539
Test recall_consolidation: 0.6971428394317627
Test recall_pneumonia: 0.7811053395271301
Test recall_atelectasis: 0.5991268754005432
Test recall_pneumothorax: 0.846001386642456
Test recall_pleural_effusion: 0.732262372970581
Test recall_pleural_other: 0.6972624659538269
Test recall_fracture: 0.20791229605674744
Test f2_score_enlarged_cardiomediastinum: 0.5232247114181519
Test f2_score_cardiomegaly: 0.746891438961029
Test f2_score_lung_opacity: 0.17138317227363586
Test f2_score_lung_lesion: 0.6645559668540955
Test f2_score_edema: 0.33830106258392334
Test f2_score_consolidation: 0.1498771607875824
Test f2_score_pneumonia: 0.47113341093063354
Test f2_score_atelectasis: 0.44856756925582886
Test f2_score_pneumothorax: 0.7966315150260925
Test f2_score_pleural_effusion: 0.0886979028582573
Test f2_score_pleural_other: 0.15950001776218414
pp = pprint.PrettyPrinter(indent=4)
if "benchmark" in data.keys():
pp.pprint(data["benchmark"]){ 'batch_size': 32,
'benchmark_name': 'Chexpert_CWBCE_E9_B32_C0_N12_D320_Masked_NoFinding',
'crop': False,
'dataset_folder': 'data/chexpert/full',
'dataset_name': 'chexpert_full',
'dim': [320, 320],
'drop_last': True,
'epochs': 9,
'label_columns': [ 'Enlarged Cardiomediastinum',
'Cardiomegaly',
'Lung Opacity',
'Lung Lesion',
'Edema',
'Consolidation',
'Pneumonia',
'Atelectasis',
'Pneumothorax',
'Pleural Effusion',
'Pleural Other',
'Fracture'],
'loss': 'weighted_binary_crossentropy',
'metrics': [ 'auc',
'precision',
'recall',
'f2_score',
'binary_accuracy',
'accuracy_enlarged_cardiomediastinum',
'accuracy_cardiomegaly',
'accuracy_lung_opacity',
'accuracy_lung_lesion',
'accuracy_edema',
'accuracy_consolidation',
'accuracy_pneumonia',
'accuracy_atelectasis',
'accuracy_pneumothorax',
'accuracy_pleural_effusion',
'accuracy_pleural_other',
'accuracy_fracture',
'auc_enlarged_cardiomediastinum',
'auc_cardiomegaly',
'auc_lung_opacity',
'auc_lung_lesion',
'auc_edema',
'auc_consolidation',
'auc_pneumonia',
'auc_atelectasis',
'auc_pneumothorax',
'auc_pleural_effusion',
'auc_pleural_other',
'auc_fracture',
'precision_enlarged_cardiomediastinum',
'precision_cardiomegaly',
'precision_lung_opacity',
'precision_lung_lesion',
'precision_edema',
'precision_consolidation',
'precision_pneumonia',
'precision_atelectasis',
'precision_pneumothorax',
'precision_pleural_effusion',
'precision_pleural_other',
'precision_fracture',
'recall_enlarged_cardiomediastinum',
'recall_cardiomegaly',
'recall_lung_opacity',
'recall_lung_lesion',
'recall_edema',
'recall_consolidation',
'recall_pneumonia',
'recall_atelectasis',
'recall_pneumothorax',
'recall_pleural_effusion',
'recall_pleural_other',
'recall_fracture',
'f2_score_enlarged_cardiomediastinum',
'f2_score_cardiomegaly',
'f2_score_lung_opacity',
'f2_score_lung_lesion',
'f2_score_edema',
'f2_score_consolidation',
'f2_score_pneumonia',
'f2_score_atelectasis',
'f2_score_pneumothorax',
'f2_score_pleural_effusion',
'f2_score_pleural_other',
'f2_score_fracture'],
'models_dir': 'models',
'n_channels': 3,
'nan_replacement': -1,
'negative_weights': [ 1.2226653099060059,
1.7059714794158936,
4.266043663024902,
1.372032642364502,
2.003906011581421,
1.2122300863265991,
1.1395792961120605,
1.5881963968276978,
1.2878180742263794,
2.468789577484131,
1.1382622718811035,
1.3658897876739502],
'optimizer': 'Adam',
'path_column': 'Path',
'path_column_prefix': '',
'positive_weights': [ 5.491044998168945,
2.416487693786621,
1.3061808347702026,
3.6879360675811768,
1.996109127998352,
5.71186637878418,
8.164386749267578,
2.7001125812530518,
4.474417209625244,
1.6808327436447144,
8.232630729675293,
3.7330634593963623],
'shuffle': True,
'split_seed': 6122156,
'test_num_samples': 44932,
'train_num_samples': 142320,
'u_enc': 'uzeroes',
'unc_value': -1,
'use_class_weights': False,
'valid_num_samples': 36162}
if 'benchmark' in data.keys() and 'split_seed' in data['benchmark']:
benchmark = data['benchmark']
dataset_path = Path(benchmark['dataset_folder'])
train_labels = benchmark['train_labels'] if 'train_labels' in benchmark.keys() else 'train.csv'
split_test_size = benchmark['split_test_size'] if 'split_test_size' in benchmark.keys() else 0.2
split_valid_size = benchmark['split_valid_size'] if 'split_valid_size' in benchmark.keys() else 0.2
split_group = benchmark['split_group'] if 'split_group' in benchmark.keys() else 'patient_id'
split_seed = benchmark['split_seed']
all_labels = pd.read_csv(dataset_path / train_labels)
train_labels, test_labels = train_test_split(all_labels, test_size=split_test_size, group=split_group, seed=split_seed)
train_labels, validation_labels = train_test_split(train_labels, test_size=split_valid_size, group=split_group, seed=split_seed)from src.datasets.u_encoding import uencode
def get_distribution(labels):
if 'nan_replacement' in benchmark.keys():
labels = labels.fillna(benchmark['nan_replacement'])
data = labels.to_numpy()
data = uencode(benchmark['u_enc'], data, unc_value=benchmark['unc_value'])
data = pd.DataFrame(data, columns=labels.columns)
labels = data[benchmark['label_columns']]
d = {'Pathology': [], 'Positive': [], 'Positive %': [], 'Negative': [], 'Negative %': [],}
for label in labels.columns:
values = labels.groupby(label)
d['Pathology'].append(label)
positive = values.size()[1.0] if 1.0 in values.size() else 0
positive_percent = positive / labels.shape[0] * 100
d['Positive'].append(positive)
d['Positive %'].append(round(positive_percent))
negative = values.size()[-0.0] if -0.0 in values.size() else 0
negative_percent = negative / labels.shape[0] * 100
d['Negative'].append(negative)
d['Negative %'].append(round(negative_percent))
df = pd.DataFrame(d)
df = df.set_index('Pathology')
return dfif 'benchmark' in data.keys() and 'split_seed' in data['benchmark']:
train = get_distribution(train_labels)
val = get_distribution(validation_labels)
test = get_distribution(test_labels)
positives = train[['Positive %']].merge(val[['Positive %']], left_index=True, right_index=True).merge(test[['Positive %']], left_index=True, right_index=True).rename(columns={"Positive %_x": "Positives Train", "Positive %_y": "Positives Validation", "Positive %": "Positives Test", })
positives.copy().plot(kind='bar', figsize=(10,7), title="Positive Labels Distribution")
negatives = train[['Negative %']].merge(val[['Negative %']], left_index=True, right_index=True).merge(test[['Negative %']], left_index=True, right_index=True).rename(columns={"Negative %_x": "Negative Train", "Negative %_y": "Negative Validation", "Negative %": "Negative Test", })
negatives.copy().plot(kind='bar', figsize=(10,7), title="Negative Labels Distribution")
train[['Positive %', 'Negative %']].copy().plot(kind='bar', figsize=(10,7), title="Training set")
val[['Positive %', 'Negative %']].copy().plot(kind='bar', figsize=(10,7), title="Validation set")
test[['Positive %', 'Negative %']].copy().plot(kind='bar', figsize=(10,7), title="Test set")
