DenseNet121_Chexpert_BCE_E5_B32_C0_N12_D256_DS9010_LR4 - TobiasSchmidtDE/DeepL-MedicalImaging GitHub Wiki
Version: 1
Trained DenseNet121 architecture using the 'Chexpert_BCE_E5_B32_C0_N12_D256_DS9010_LR4' benchmark. The benchmark was initialized for the chexpert_full dataset with batch size of 32, shuffle set to True and images rescaled to dimension (256, 256). The training was done for 5 epochs using the Adam optimizer and binary_crossentropy loss. A total of 12 labels/pathologies were included in the training and encoded using the 'uzeroes' method. The traing set included 200278 number of sample, the validation set 234, and the test set 234.
from pathlib import Path
from dotenv import load_dotenv, find_dotenv
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.pyplot import figure
import json
import os
import re
import pprint
basepath = Path(os.getcwd())
if basepath.name != "idp-radio-1":
os.chdir(basepath.parent.parent)
print(os.getcwd())
load_dotenv(find_dotenv())
from src.preprocessing.split.train_test_split import train_test_split/srv/idp-radio-1
data = json.loads(os.environ['EXP_DATA'])
history = data['history']for s in data["description"].split(".")[:-1]:
print(s + ".\n")Trained DenseNet121 architecture using the 'Chexpert_BCE_E5_B32_C0_N12_D256_DS9010_LR4' benchmark.
The benchmark was initialized for the chexpert_full dataset with batch size of 32, shuffle set to True and images rescaled to dimension (256, 256).
The training was done for 5 epochs using the Adam optimizer and binary_crossentropy loss.
A total of 12 labels/pathologies were included in the training and encoded using the 'uzeroes' method.
The traing set included 200278 number of sample, the validation set 234, and the test set 234.
# if there are any metrics that were renamed, add this new name here as ("default_name":"new_name")
metric_custom_names={"auc":"AUC_ROC"}
metric_names = [re.sub("([a-z0-9])([A-Z])","\g<1> \g<2>",name) for name in data["benchmark"]["metrics"]]
metric_keys = [re.sub("([a-z0-9])([A-Z])","\g<1>_\g<2>",name).lower() for name in data["benchmark"]["metrics"]]
for default_name, custom_name in metric_custom_names.items():
if not default_name in history.keys() and default_name in metric_keys:
#replace default name with custom name
metric_keys[metric_keys.index(default_name)]=custom_namedef print_or_plot_metric(metric_key, metric_name, figure_name):
if len(history[metric_key]) == 1:
print("Data for {m_name} only available for a single epoch. \nSkipping plot and printing data...".format(m_name=metric_name))
print('Train {}: '.format(metric_name), history[metric_key])
print('Validation {}: '.format(metric_name), history['val_'+metric_key])
print()
else:
plot_epoch_metric(metric_key, metric_name, figure_name)
def plot_epoch_metric(metric_key, metric_name, figure_name):
figure(num=None, figsize=(10, 6))
plt.plot(history[metric_key])
if 'val_'+metric_key in history.keys():
plt.plot(history['val_'+metric_key])
plt.title(figure_name)
plt.ylabel(metric_name)
plt.xlabel('Epoch')
if 'val_'+metric_key in history.keys():
plt.legend(['Train', 'Validation'], loc='upper left')
plt.show()
for i, metric_key in enumerate(metric_keys):
print_or_plot_metric(metric_key, metric_names[i], "Model "+metric_names[i])
print_or_plot_metric("loss", "Loss", "Model loss")
if "lr" in history.keys():
plot_epoch_metric("lr", "Learning Rate", "Learning Rate")
if 'classification_report' in data.keys() and data['classification_report']:
print(data['classification_report']) precision recall f1-score support
Enlarged Cardiomediastinum 1.00 0.01 0.02 109
Cardiomegaly 0.29 0.06 0.10 68
Lung Opacity 0.50 0.43 0.46 126
Lung Lesion 0.00 0.00 0.00 1
Edema 0.15 0.09 0.11 45
Consolidation 0.00 0.00 0.00 33
Pneumonia 0.00 0.00 0.00 8
Atelectasis 0.00 0.00 0.00 80
Pneumothorax 0.14 0.12 0.13 8
Pleural Effusion 0.31 0.27 0.29 67
Pleural Other 0.00 0.00 0.00 1
Fracture 0.00 0.00 0.00 0
micro avg 0.37 0.15 0.21 546
macro avg 0.20 0.08 0.09 546
weighted avg 0.40 0.15 0.17 546
samples avg 0.22 0.09 0.12 546
if 'test' in data.keys() and data['test']:
for score_name, score in data["test"].items():
print('Test {}: '.format(score_name), score)
Test loss: 0.40732863545417786
Test auc: 0.746934175491333
Test precision: 0.8206278085708618
Test recall: 0.3351648449897766
Test f2_score: 0.3801412880420685
Test binary_accuracy: 0.8564814329147339
Test accuracy_enlarged_cardiomediastinum: 0.5384615659713745
Test accuracy_cardiomegaly: 0.7606837749481201
Test accuracy_lung_opacity: 0.8162392973899841
Test accuracy_lung_lesion: 0.995726466178894
Test accuracy_edema: 0.8547008633613586
Test accuracy_consolidation: 0.867521345615387
Test accuracy_pneumonia: 0.9658119678497314
Test accuracy_atelectasis: 0.6623931527137756
Test accuracy_pneumothorax: 0.9615384340286255
Test accuracy_pleural_effusion: 0.8632478713989258
Test accuracy_pleural_other: 0.9914529919624329
Test accuracy_fracture: 1.0
Test auc_enlarged_cardiomediastinum: 0.5799633264541626
Test auc_cardiomegaly: 0.8330084681510925
Test auc_lung_opacity: 0.9099794626235962
Test auc_lung_lesion: 0.8047210574150085
Test auc_edema: 0.8948265314102173
Test auc_consolidation: 0.9046434760093689
Test auc_pneumonia: 0.7419800758361816
Test auc_atelectasis: 0.7084820866584778
Test auc_pneumothorax: 0.7159845232963562
Test auc_pleural_effusion: 0.9318526387214661
Test auc_pleural_other: 0.9377682209014893
Test auc_fracture: 0.0
Test precision_enlarged_cardiomediastinum: 1.0
Test precision_cardiomegaly: 0.9285714030265808
Test precision_lung_opacity: 0.8807339668273926
Test precision_lung_lesion: 0.0
Test precision_edema: 0.7037037014961243
Test precision_consolidation: 0.75
Test precision_pneumonia: 0.0
Test precision_atelectasis: 1.0
Test precision_pneumothorax: 0.4285714328289032
Test precision_pleural_effusion: 0.7966101765632629
Test precision_pleural_other: 0.0
Test precision_fracture: 0.0
Test recall_enlarged_cardiomediastinum: 0.00917431153357029
Test recall_cardiomegaly: 0.19117647409439087
Test recall_lung_opacity: 0.761904776096344
Test recall_lung_lesion: 0.0
Test recall_edema: 0.42222222685813904
Test recall_consolidation: 0.09090909361839294
Test recall_pneumonia: 0.0
Test recall_atelectasis: 0.012500000186264515
Test recall_pneumothorax: 0.375
Test recall_pleural_effusion: 0.7014925479888916
Test recall_pleural_other: 0.0
Test recall_fracture: 0.0
Test f2_score_enlarged_cardiomediastinum: 0.011441647075116634
Test f2_score_cardiomegaly: 0.22727273404598236
Test f2_score_lung_opacity: 0.7830342650413513
Test f2_score_lung_lesion: 0.0
Test f2_score_edema: 0.45893722772598267
Test f2_score_consolidation: 0.11029411852359772
Test f2_score_pneumonia: 0.0
Test f2_score_atelectasis: 0.015576324425637722
Test f2_score_pneumothorax: 0.38461536169052124
Test f2_score_pleural_effusion: 0.7186543941497803
Test f2_score_pleural_other: 0.0
Test f2_score_fracture: 0.0
pp = pprint.PrettyPrinter(indent=4)
if "benchmark" in data.keys():
pp.pprint(data["benchmark"]){ 'batch_size': 32,
'benchmark_name': 'Chexpert_BCE_E5_B32_C0_N12_D256_DS9010_LR4',
'crop': False,
'dataset_folder': 'data/chexpert/full',
'dataset_name': 'chexpert_full',
'dim': [256, 256],
'drop_last': True,
'epochs': 5,
'label_columns': [ 'Enlarged Cardiomediastinum',
'Cardiomegaly',
'Lung Opacity',
'Lung Lesion',
'Edema',
'Consolidation',
'Pneumonia',
'Atelectasis',
'Pneumothorax',
'Pleural Effusion',
'Pleural Other',
'Fracture'],
'learning_rate': 0.0001,
'loss': 'binary_crossentropy',
'metrics': [ 'auc',
'precision',
'recall',
'f2_score',
'binary_accuracy',
'accuracy_enlarged_cardiomediastinum',
'accuracy_cardiomegaly',
'accuracy_lung_opacity',
'accuracy_lung_lesion',
'accuracy_edema',
'accuracy_consolidation',
'accuracy_pneumonia',
'accuracy_atelectasis',
'accuracy_pneumothorax',
'accuracy_pleural_effusion',
'accuracy_pleural_other',
'accuracy_fracture',
'auc_enlarged_cardiomediastinum',
'auc_cardiomegaly',
'auc_lung_opacity',
'auc_lung_lesion',
'auc_edema',
'auc_consolidation',
'auc_pneumonia',
'auc_atelectasis',
'auc_pneumothorax',
'auc_pleural_effusion',
'auc_pleural_other',
'auc_fracture',
'precision_enlarged_cardiomediastinum',
'precision_cardiomegaly',
'precision_lung_opacity',
'precision_lung_lesion',
'precision_edema',
'precision_consolidation',
'precision_pneumonia',
'precision_atelectasis',
'precision_pneumothorax',
'precision_pleural_effusion',
'precision_pleural_other',
'precision_fracture',
'recall_enlarged_cardiomediastinum',
'recall_cardiomegaly',
'recall_lung_opacity',
'recall_lung_lesion',
'recall_edema',
'recall_consolidation',
'recall_pneumonia',
'recall_atelectasis',
'recall_pneumothorax',
'recall_pleural_effusion',
'recall_pleural_other',
'recall_fracture',
'f2_score_enlarged_cardiomediastinum',
'f2_score_cardiomegaly',
'f2_score_lung_opacity',
'f2_score_lung_lesion',
'f2_score_edema',
'f2_score_consolidation',
'f2_score_pneumonia',
'f2_score_atelectasis',
'f2_score_pneumothorax',
'f2_score_pleural_effusion',
'f2_score_pleural_other',
'f2_score_fracture'],
'models_dir': 'models',
'n_channels': 3,
'nan_replacement': 0,
'negative_weights': [ 1.050765872001648,
1.1362944841384888,
1.8964533805847168,
1.0425058603286743,
1.3045483827590942,
1.070893406867981,
1.027823567390442,
1.1751078367233276,
1.0955462455749512,
1.6264314651489258,
1.0160895586013794,
1.0419689416885376],
'optimizer': 'Adam',
'path_column': 'Path',
'path_column_prefix': '',
'positive_weights': [ 20.698293685913086,
8.337052345275879,
2.115506887435913,
24.526147842407227,
4.28355073928833,
15.105680465698242,
36.9407844543457,
6.7107672691345215,
11.466132164001465,
2.596343755722046,
63.15231704711914,
24.82717514038086],
'shuffle': True,
'split_seed': 6122156,
'test_num_samples': 234,
'train_num_samples': 200278,
'u_enc': 'uzeroes',
'unc_value': -1,
'use_class_weights': False,
'valid_num_samples': 234}
if 'benchmark' in data.keys() and 'split_seed' in data['benchmark']:
benchmark = data['benchmark']
dataset_path = Path(benchmark['dataset_folder'])
train_labels = benchmark['train_labels'] if 'train_labels' in benchmark.keys() else 'train.csv'
test_labels = benchmark['test_labels'] if 'test_labels' in benchmark.keys() else None
split_test_size = benchmark['split_test_size'] if 'split_test_size' in benchmark.keys() else 0.1
split_valid_size = benchmark['split_valid_size'] if 'split_valid_size' in benchmark.keys() else 0.1
split_group = benchmark['split_group'] if 'split_group' in benchmark.keys() else 'patient_id'
split_seed = benchmark['split_seed']
if test_labels is None:
# read all labels from one file and split into train/test/valid
all_labels = pd.read_csv(dataset_path / train_labels)
train_labels, test_labels = train_test_split(
all_labels, test_size=split_test_size, group=split_group, seed=split_seed)
train_labels, validation_labels = train_test_split(
train_labels, test_size=split_valid_size, group=split_group, seed=split_seed)
else:
# read train and valid labels from one file and test from another.
train_labels = pd.read_csv(dataset_path / train_labels)
train_labels, validation_labels = train_test_split(
train_labels, test_size=split_valid_size, group=split_group, seed=split_seed)
test_labels = pd.read_csv(dataset_path / test_labels)from src.datasets.u_encoding import uencode
def get_distribution(labels):
if 'nan_replacement' in benchmark.keys():
labels = labels.fillna(benchmark['nan_replacement'])
data = labels.to_numpy()
data = uencode(benchmark['u_enc'], data, unc_value=benchmark['unc_value'])
data = pd.DataFrame(data, columns=labels.columns)
labels = data[benchmark['label_columns']]
d = {'Pathology': [], 'Positive': [], 'Positive %': [], 'Negative': [], 'Negative %': [],}
for label in labels.columns:
values = labels.groupby(label)
d['Pathology'].append(label)
positive = values.size()[1.0] if 1.0 in values.size() else 0
positive_percent = positive / labels.shape[0] * 100
d['Positive'].append(positive)
d['Positive %'].append(round(positive_percent))
negative = values.size()[-0.0] if -0.0 in values.size() else 0
negative_percent = negative / labels.shape[0] * 100
d['Negative'].append(negative)
d['Negative %'].append(round(negative_percent))
df = pd.DataFrame(d)
df = df.set_index('Pathology')
return dfif 'benchmark' in data.keys() and 'split_seed' in data['benchmark']:
train = get_distribution(train_labels)
val = get_distribution(validation_labels)
test = get_distribution(test_labels)
positives = train[['Positive %']].merge(val[['Positive %']], left_index=True, right_index=True).merge(test[['Positive %']], left_index=True, right_index=True).rename(columns={"Positive %_x": "Positives Train", "Positive %_y": "Positives Validation", "Positive %": "Positives Test", })
positives.copy().plot(kind='bar', figsize=(10,7), title="Positive Labels Distribution")
negatives = train[['Negative %']].merge(val[['Negative %']], left_index=True, right_index=True).merge(test[['Negative %']], left_index=True, right_index=True).rename(columns={"Negative %_x": "Negative Train", "Negative %_y": "Negative Validation", "Negative %": "Negative Test", })
negatives.copy().plot(kind='bar', figsize=(10,7), title="Negative Labels Distribution")
train[['Positive %', 'Negative %']].copy().plot(kind='bar', figsize=(10,7), title="Training set")
val[['Positive %', 'Negative %']].copy().plot(kind='bar', figsize=(10,7), title="Validation set")
test[['Positive %', 'Negative %']].copy().plot(kind='bar', figsize=(10,7), title="Test set")
