# Load with automatic adapter detection
adata = data_manager.load_modality(
    name="geo_gse12345",
    source="path/to/data.h5ad",
    adapter="transcriptomics_single_cell",
    validate=True
)

# Load with metadata enrichment
adata = data_manager.load_modality(
    name="proteomics_sample",
    source=dataframe,
    adapter="proteomics_ms",
    sample_metadata=metadata_df,
    experimental_design="case_control"
)

# Get specific modality
adata = data_manager.get_modality("geo_gse12345_clustered")

# List all modalities
modalities = data_manager.list_modalities()

# Get quality metrics
metrics = data_manager.get_quality_metrics("geo_gse12345")

# Save individual modality
path = data_manager.save_modality(
    name="geo_gse12345_processed",
    path="processed_data.h5ad"
)

# Save multi-modal dataset
path = data_manager.save_mudata(
    path="integrated_analysis.h5mu",
    modalities=["rna_seq", "proteomics"]
)

class IModalityAdapter(ABC):
    @abstractmethod
    def from_source(self, source: Any, **kwargs) -> anndata.AnnData:
        """Load data from source and convert to AnnData."""
        pass

    @abstractmethod
    def validate(self, adata: anndata.AnnData, strict: bool = False) -> ValidationResult:
        """Validate AnnData against schema."""
        pass

    @abstractmethod
    def get_quality_metrics(self, adata: anndata.AnnData) -> Dict[str, Any]:
        """Calculate quality control metrics."""
        pass

# Single-cell RNA-seq
adapter = TranscriptomicsAdapter(
    data_type="single_cell",
    strict_validation=False
)

adata = adapter.from_source(
    source="counts.csv",
    transpose=True,
    gene_symbols_col="gene_name",
    sample_metadata=metadata_df
)

# Bulk RNA-seq
adapter = TranscriptomicsAdapter(
    data_type="bulk",
    strict_validation=True
)

adata = adapter.from_source(
    source=count_matrix,
    experimental_design="factorial",
    batch_correction_method="combat"
)

# Mass spectrometry proteomics
adapter = ProteomicsAdapter(
    data_type="mass_spectrometry",
    strict_validation=False
)

adata = adapter.from_source(
    source="proteinGroups.txt",
    intensity_columns=["LFQ intensity"],
    missing_value_threshold=0.7,
    normalize_method="tmm"
)

# Affinity proteomics
adapter = ProteomicsAdapter(
    data_type="affinity",
    strict_validation=True
)

adata = adapter.from_source(
    source="olink_data.csv",
    npx_columns=True,
    panel_info=panel_metadata,
    cv_threshold=0.15
)

class IDataBackend(ABC):
    @abstractmethod
    def save(self, data: anndata.AnnData, path: Union[str, Path], **kwargs) -> str:
        """Save AnnData to storage."""
        pass

    @abstractmethod
    def load(self, path: Union[str, Path], **kwargs) -> anndata.AnnData:
        """Load AnnData from storage."""
        pass

    @abstractmethod
    def get_storage_info(self) -> Dict[str, Any]:
        """Get backend storage information."""
        pass

h5ad_backend = H5ADBackend(
    base_path=workspace_path / "data",
    compression="gzip",
    compression_opts=6
)

# Future S3 configuration (prepared)
h5ad_backend.s3_config = {
    "bucket": "lobster-data",
    "region": "us-west-2",
    "access_key": "key",
    "secret_key": "secret"
}

def _sanitize_dataframe_for_h5ad(df: pd.DataFrame) -> pd.DataFrame:
    """Sanitize metadata for H5AD serialization."""

    # Convert ArrowExtensionArray columns (e.g., string[pyarrow])
    for col in df.columns:
        if isinstance(df[col].dtype, pd.ArrowDtype):
            df[col] = df[col].astype(str)

    # Sanitize values in nested structures (adata.uns)
    for key, value in adata.uns.items():
        if isinstance(value, dict):
            adata.uns[key] = sanitize_nested_dict(value)

    return df

def sanitize_nested_dict(obj):
    """Aggressive stringification for H5AD compatibility."""

    # Convert int/float to strings
    if isinstance(obj, (int, float)):
        return str(obj)

    # Convert lists to numpy string arrays
    if isinstance(obj, list):
        return np.array([str(item) for item in obj], dtype=str)

    # Recursively sanitize dicts
    if isinstance(obj, dict):
        return {k.replace('/', '__'): sanitize_nested_dict(v)
                for k, v in obj.items()}

    return obj

# Create integrated dataset
mdata = data_manager.to_mudata(
    modalities=["rna_seq", "proteomics", "metabolomics"]
)

# Save multi-modal analysis
path = data_manager.save_mudata(
    path="integrated_multiomics.h5mu",
    modalities=["processed_rna", "processed_proteins"]
)

# Validation result structure
@dataclass
class ValidationResult:
    is_valid: bool
    errors: List[str]          # Critical issues
    warnings: List[str]        # Recommendations
    suggestions: List[str]     # Best practices
    compliance_score: float    # 0.0 to 1.0

class ProvenanceTracker:
    def __init__(self):
        self.activities: Dict[str, Activity] = {}
        self.entities: Dict[str, Entity] = {}
        self.agents: Dict[str, Agent] = {}

    def log_data_loading(self, source_path, output_entity_id, adapter_name, parameters):
        """Log data loading activity."""
        activity_id = self.create_activity(
            activity_type="data_loading",
            started_at=datetime.now(),
            parameters=parameters
        )

        # Link source to output
        self.add_derivation(
            derived_entity_id=output_entity_id,
            source_entity_id=source_path,
            activity_id=activity_id
        )

# Automatic logging in agent tools
data_manager.log_tool_usage(
    tool_name="cluster_modality",
    parameters={
        "resolution": 0.5,
        "algorithm": "leiden",
        "random_state": 42
    },
    description="Performed Leiden clustering on preprocessed data"
)

# Session metadata structure
{
    "session_id": "unique_identifier",
    "created_at": "2024-01-15T10:30:00",
    "last_modified": "2024-01-15T15:45:00",
    "lobster_version": "2.0.0",
    "active_modalities": {
        "geo_gse12345": {
            "path": "/data/geo_gse12345.h5ad",
            "shape": [5000, 20000],
            "last_accessed": "2024-01-15T15:30:00"
        }
    },
    "workspace_stats": {
        "total_datasets": 5,
        "total_loaded": 2,
        "total_size_mb": 150.5
    }
}

# Workspace scanning
data_manager._scan_workspace()  # Automatic on startup

# Dataset restoration
result = data_manager.restore_session(
    pattern="recent",  # or "all", or glob pattern
    max_size_mb=1000
)

# Automatic state saving
saved_items = data_manager.auto_save_state()

def _match_modality_to_adapter(self, modality_name: str) -> Optional[str]:
    """Smart matching of modality names to adapter names."""
    adapter_keywords = {
        'transcriptomics_single_cell': {
            'primary': ['single_cell', 'single-cell', 'sc_', 'scrna', '10x'],
            'secondary': ['transcriptom', 'rna', 'gene_expression'],
            'exclude': ['bulk']
        },
        'proteomics_ms': {
            'primary': ['ms', 'mass_spec', 'mass_spectrometry'],
            'secondary': ['proteomic', 'protein'],
            'exclude': ['affinity', 'antibody']
        }
    }
    # Scoring algorithm for best match...