Temporal Knowledge Graph - chunhualiao/public-docs GitHub Wiki

graph LR
  subgraph UserQuery["User Query"]
    A[Image]
    B[Text / Speech]
    C[GPS + Time<br/>Metadata]
  end

  subgraph UnimodalEncoders["Uni-modal Encoders"]
    A --> V[Vision Encoder]
    B --> T[Text Encoder]
    C --> M[Metadata Embedder]
  end

  subgraph MultimodalTransformer["Multimodal Transformer Model<br/>early-concat self-attention"]
    V --> F[JointFeat<br/>fusion]
    T --> F
    M --> F
  end

  subgraph TemporalKnowledgeGraph["Temporal Knowledge Graph"]
    K[TKG Lookup & Embedding]
  end
  
  F -->|contextual query| K
  K -->|enriched triples| F

  subgraph ReasoningRanking["Reasoning & Ranking"]
    R[α·MTM + β·NLP + γ·TKG<br/>Table 1 scoring]
  end
  
  F --> R

  subgraph DecoderLLM["Decoder-only LLM / NLP"]
    D[Generative Decoder]
  end
  
  R --> D
  D --> O[Natural-language Answer<br/>+ maps / routes]

  O -.->|sensor updates,<br/>user ratings| V
  O -.-> K

  %% Style definitions
  classDef mtm fill:#ffe0cc,stroke:#c37b00
  classDef tkg fill:#e0ecff,stroke:#3366cc
  classDef rank fill:#e7ffe7,stroke:#2e8b57
  classDef dec fill:#fff6d6,stroke:#d4a017

  %% Apply styles
  class F mtm
  class K tkg
  class R rank
  class D dec