DenseNet121_Chexpert_BCE_E3_B32_C0_N12_D256_DS9505_2LR4 - TobiasSchmidtDE/DeepL-MedicalImaging GitHub Wiki
Version: 1
Trained DenseNet121 architecture using the 'Chexpert_BCE_E3_B32_C0_N12_D256_DS9505_2LR4' benchmark. The benchmark was initialized for the chexpert_full dataset with batch size of 32, shuffle set to True and images rescaled to dimension (256, 256). The training was done for 3 epochs using the Adam optimizer and binary_crossentropy loss. A total of 12 labels/pathologies were included in the training and encoded using the 'uzeroes' method. The traing set included 211818 number of sample, the validation set 11596, and the test set 234.
from pathlib import Path
from dotenv import load_dotenv, find_dotenv
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.pyplot import figure
import json
import os
import re
import pprint
basepath = Path(os.getcwd())
if basepath.name != "idp-radio-1":
os.chdir(basepath.parent.parent)
print(os.getcwd())
load_dotenv(find_dotenv())
from src.preprocessing.split.train_test_split import train_test_split/srv/idp-radio-1
data = json.loads(os.environ['EXP_DATA'])
history = data['history']for s in data["description"].split(".")[:-1]:
print(s + ".\n")Trained DenseNet121 architecture using the 'Chexpert_BCE_E3_B32_C0_N12_D256_DS9505_2LR4' benchmark.
The benchmark was initialized for the chexpert_full dataset with batch size of 32, shuffle set to True and images rescaled to dimension (256, 256).
The training was done for 3 epochs using the Adam optimizer and binary_crossentropy loss.
A total of 12 labels/pathologies were included in the training and encoded using the 'uzeroes' method.
The traing set included 211818 number of sample, the validation set 11596, and the test set 234.
# if there are any metrics that were renamed, add this new name here as ("default_name":"new_name")
metric_custom_names={"auc":"AUC_ROC"}
metric_names = [re.sub("([a-z0-9])([A-Z])","\g<1> \g<2>",name) for name in data["benchmark"]["metrics"]]
metric_keys = [re.sub("([a-z0-9])([A-Z])","\g<1>_\g<2>",name).lower() for name in data["benchmark"]["metrics"]]
for default_name, custom_name in metric_custom_names.items():
if not default_name in history.keys() and default_name in metric_keys:
#replace default name with custom name
metric_keys[metric_keys.index(default_name)]=custom_namedef print_or_plot_metric(metric_key, metric_name, figure_name):
if len(history[metric_key]) == 1:
print("Data for {m_name} only available for a single epoch. \nSkipping plot and printing data...".format(m_name=metric_name))
print('Train {}: '.format(metric_name), history[metric_key])
print('Validation {}: '.format(metric_name), history['val_'+metric_key])
print()
else:
plot_epoch_metric(metric_key, metric_name, figure_name)
def plot_epoch_metric(metric_key, metric_name, figure_name):
figure(num=None, figsize=(10, 6))
plt.plot(history[metric_key])
if 'val_'+metric_key in history.keys():
plt.plot(history['val_'+metric_key])
plt.title(figure_name)
plt.ylabel(metric_name)
plt.xlabel('Epoch')
if 'val_'+metric_key in history.keys():
plt.legend(['Train', 'Validation'], loc='upper left')
plt.show()
for i, metric_key in enumerate(metric_keys):
print_or_plot_metric(metric_key, metric_names[i], "Model "+metric_names[i])
print_or_plot_metric("loss", "Loss", "Model loss")
if "lr" in history.keys():
plot_epoch_metric("lr", "Learning Rate", "Learning Rate")
if 'classification_report' in data.keys() and data['classification_report']:
print(data['classification_report']) precision recall f1-score support
Enlarged Cardiomediastinum 0.00 0.00 0.00 109
Cardiomegaly 0.00 0.00 0.00 68
Lung Opacity 0.54 1.00 0.70 126
Lung Lesion 0.00 0.00 0.00 1
Edema 0.00 0.00 0.00 45
Consolidation 0.00 0.00 0.00 33
Pneumonia 0.00 0.00 0.00 8
Atelectasis 0.00 0.00 0.00 80
Pneumothorax 0.00 0.00 0.00 8
Pleural Effusion 0.00 0.00 0.00 67
Pleural Other 0.00 0.00 0.00 1
Fracture 0.00 0.00 0.00 0
micro avg 0.54 0.23 0.32 546
macro avg 0.04 0.08 0.06 546
weighted avg 0.12 0.23 0.16 546
samples avg 0.54 0.17 0.25 546
if 'test' in data.keys() and data['test']:
for score_name, score in data["test"].items():
print('Test {}: '.format(score_name), score)
Test loss: 0.6902080178260803
Test auc: 0.4583333432674408
Test precision: 0.5384615659713745
Test recall: 0.23076923191547394
Test f2_score: 0.26054590940475464
Test binary_accuracy: 0.8119661211967468
Test accuracy_enlarged_cardiomediastinum: 0.5341880321502686
Test accuracy_cardiomegaly: 0.7094017267227173
Test accuracy_lung_opacity: 0.5384615659713745
Test accuracy_lung_lesion: 0.995726466178894
Test accuracy_edema: 0.807692289352417
Test accuracy_consolidation: 0.8589743375778198
Test accuracy_pneumonia: 0.9658119678497314
Test accuracy_atelectasis: 0.6581196784973145
Test accuracy_pneumothorax: 0.9658119678497314
Test accuracy_pleural_effusion: 0.7136752009391785
Test accuracy_pleural_other: 0.995726466178894
Test accuracy_fracture: 1.0
Test auc_enlarged_cardiomediastinum: 0.5
Test auc_cardiomegaly: 0.5
Test auc_lung_opacity: 0.5
Test auc_lung_lesion: 0.5
Test auc_edema: 0.5
Test auc_consolidation: 0.5
Test auc_pneumonia: 0.5
Test auc_atelectasis: 0.5
Test auc_pneumothorax: 0.5
Test auc_pleural_effusion: 0.5
Test auc_pleural_other: 0.5
Test auc_fracture: 0.0
Test precision_enlarged_cardiomediastinum: 0.0
Test precision_cardiomegaly: 0.0
Test precision_lung_opacity: 0.5384615659713745
Test precision_lung_lesion: 0.0
Test precision_edema: 0.0
Test precision_consolidation: 0.0
Test precision_pneumonia: 0.0
Test precision_atelectasis: 0.0
Test precision_pneumothorax: 0.0
Test precision_pleural_effusion: 0.0
Test precision_pleural_other: 0.0
Test precision_fracture: 0.0
Test recall_enlarged_cardiomediastinum: 0.0
Test recall_cardiomegaly: 0.0
Test recall_lung_opacity: 1.0
Test recall_lung_lesion: 0.0
Test recall_edema: 0.0
Test recall_consolidation: 0.0
Test recall_pneumonia: 0.0
Test recall_atelectasis: 0.0
Test recall_pneumothorax: 0.0
Test recall_pleural_effusion: 0.0
Test recall_pleural_other: 0.0
Test recall_fracture: 0.0
Test f2_score_enlarged_cardiomediastinum: 0.0
Test f2_score_cardiomegaly: 0.0
Test f2_score_lung_opacity: 0.8536585569381714
Test f2_score_lung_lesion: 0.0
Test f2_score_edema: 0.0
Test f2_score_consolidation: 0.0
Test f2_score_pneumonia: 0.0
Test f2_score_atelectasis: 0.0
Test f2_score_pneumothorax: 0.0
Test f2_score_pleural_effusion: 0.0
Test f2_score_pleural_other: 0.0
Test f2_score_fracture: 0.0
pp = pprint.PrettyPrinter(indent=4)
if "benchmark" in data.keys():
pp.pprint(data["benchmark"]){ 'augmentation': None,
'batch_size': 32,
'benchmark_name': 'Chexpert_BCE_E3_B32_C0_N12_D256_DS9505_2LR4',
'crop': False,
'dataset_folder': 'data/chexpert/full',
'dataset_name': 'chexpert_full',
'dim': [256, 256],
'drop_last': False,
'epochs': 3,
'label_columns': [ 'Enlarged Cardiomediastinum',
'Cardiomegaly',
'Lung Opacity',
'Lung Lesion',
'Edema',
'Consolidation',
'Pneumonia',
'Atelectasis',
'Pneumothorax',
'Pleural Effusion',
'Pleural Other',
'Fracture'],
'learning_rate': 0.0001,
'loss': 'binary_crossentropy',
'metrics': [ 'auc',
'precision',
'recall',
'f2_score',
'binary_accuracy',
'accuracy_enlarged_cardiomediastinum',
'accuracy_cardiomegaly',
'accuracy_lung_opacity',
'accuracy_lung_lesion',
'accuracy_edema',
'accuracy_consolidation',
'accuracy_pneumonia',
'accuracy_atelectasis',
'accuracy_pneumothorax',
'accuracy_pleural_effusion',
'accuracy_pleural_other',
'accuracy_fracture',
'auc_enlarged_cardiomediastinum',
'auc_cardiomegaly',
'auc_lung_opacity',
'auc_lung_lesion',
'auc_edema',
'auc_consolidation',
'auc_pneumonia',
'auc_atelectasis',
'auc_pneumothorax',
'auc_pleural_effusion',
'auc_pleural_other',
'auc_fracture',
'precision_enlarged_cardiomediastinum',
'precision_cardiomegaly',
'precision_lung_opacity',
'precision_lung_lesion',
'precision_edema',
'precision_consolidation',
'precision_pneumonia',
'precision_atelectasis',
'precision_pneumothorax',
'precision_pleural_effusion',
'precision_pleural_other',
'precision_fracture',
'recall_enlarged_cardiomediastinum',
'recall_cardiomegaly',
'recall_lung_opacity',
'recall_lung_lesion',
'recall_edema',
'recall_consolidation',
'recall_pneumonia',
'recall_atelectasis',
'recall_pneumothorax',
'recall_pleural_effusion',
'recall_pleural_other',
'recall_fracture',
'f2_score_enlarged_cardiomediastinum',
'f2_score_cardiomegaly',
'f2_score_lung_opacity',
'f2_score_lung_lesion',
'f2_score_edema',
'f2_score_consolidation',
'f2_score_pneumonia',
'f2_score_atelectasis',
'f2_score_pneumothorax',
'f2_score_pleural_effusion',
'f2_score_pleural_other',
'f2_score_fracture'],
'models_dir': 'models',
'n_channels': 3,
'nan_replacement': 0,
'negative_weights': [ 1.0508148670196533,
1.136679768562317,
1.8971607685089111,
1.0428425073623657,
1.3051981925964355,
1.0711511373519897,
1.0278136730194092,
1.1755650043487549,
1.0955102443695068,
1.6276289224624634,
1.0160306692123413,
1.0420267581939697],
'num_samples_test': 234,
'num_samples_train': 211818,
'num_samples_validation': 11596,
'optimizer': 'Adam',
'path_column': 'Path',
'path_column_prefix': '',
'positive_weights': [ 20.679292678833008,
8.31637191772461,
2.1146273612976074,
24.34130096435547,
4.276559829711914,
15.054584503173828,
36.95359420776367,
6.695896625518799,
11.470081329345703,
2.5932981967926025,
63.380611419677734,
24.794334411621094],
'shuffle': True,
'split_seed': 6122156,
'u_enc': 'uzeroes',
'unc_value': -1,
'use_class_weights': False}
if 'benchmark' in data.keys() and 'split_seed' in data['benchmark']:
benchmark = data['benchmark']
dataset_path = Path(benchmark['dataset_folder'])
train_labels = benchmark['train_labels'] if 'train_labels' in benchmark.keys() else 'train.csv'
test_labels = benchmark['test_labels'] if 'test_labels' in benchmark.keys() else None
split_test_size = benchmark['split_test_size'] if 'split_test_size' in benchmark.keys() else 0.1
split_valid_size = benchmark['split_valid_size'] if 'split_valid_size' in benchmark.keys() else 0.1
split_group = benchmark['split_group'] if 'split_group' in benchmark.keys() else 'patient_id'
split_seed = benchmark['split_seed']
if test_labels is None:
# read all labels from one file and split into train/test/valid
all_labels = pd.read_csv(dataset_path / train_labels)
train_labels, test_labels = train_test_split(
all_labels, test_size=split_test_size, group=split_group, seed=split_seed)
train_labels, validation_labels = train_test_split(
train_labels, test_size=split_valid_size, group=split_group, seed=split_seed)
else:
# read train and valid labels from one file and test from another.
train_labels = pd.read_csv(dataset_path / train_labels)
train_labels, validation_labels = train_test_split(
train_labels, test_size=split_valid_size, group=split_group, seed=split_seed)
test_labels = pd.read_csv(dataset_path / test_labels)from src.datasets.u_encoding import uencode
def get_distribution(labels):
if 'nan_replacement' in benchmark.keys():
labels = labels.fillna(benchmark['nan_replacement'])
data = labels.to_numpy()
data = uencode(benchmark['u_enc'], data, unc_value=benchmark['unc_value'])
data = pd.DataFrame(data, columns=labels.columns)
labels = data[benchmark['label_columns']]
d = {'Pathology': [], 'Positive': [], 'Positive %': [], 'Negative': [], 'Negative %': [],}
for label in labels.columns:
values = labels.groupby(label)
d['Pathology'].append(label)
positive = values.size()[1.0] if 1.0 in values.size() else 0
positive_percent = positive / labels.shape[0] * 100
d['Positive'].append(positive)
d['Positive %'].append(round(positive_percent))
negative = values.size()[-0.0] if -0.0 in values.size() else 0
negative_percent = negative / labels.shape[0] * 100
d['Negative'].append(negative)
d['Negative %'].append(round(negative_percent))
df = pd.DataFrame(d)
df = df.set_index('Pathology')
return dfif 'benchmark' in data.keys() and 'split_seed' in data['benchmark']:
train = get_distribution(train_labels)
val = get_distribution(validation_labels)
test = get_distribution(test_labels)
positives = train[['Positive %']].merge(val[['Positive %']], left_index=True, right_index=True).merge(test[['Positive %']], left_index=True, right_index=True).rename(columns={"Positive %_x": "Positives Train", "Positive %_y": "Positives Validation", "Positive %": "Positives Test", })
positives.copy().plot(kind='bar', figsize=(10,7), title="Positive Labels Distribution")
negatives = train[['Negative %']].merge(val[['Negative %']], left_index=True, right_index=True).merge(test[['Negative %']], left_index=True, right_index=True).rename(columns={"Negative %_x": "Negative Train", "Negative %_y": "Negative Validation", "Negative %": "Negative Test", })
negatives.copy().plot(kind='bar', figsize=(10,7), title="Negative Labels Distribution")
train[['Positive %', 'Negative %']].copy().plot(kind='bar', figsize=(10,7), title="Training set")
val[['Positive %', 'Negative %']].copy().plot(kind='bar', figsize=(10,7), title="Validation set")
test[['Positive %', 'Negative %']].copy().plot(kind='bar', figsize=(10,7), title="Test set")
