Evaluate topics

[RFOxbury]

Provide quantitative evaluation of the topics generated with BERTopic.

From Shabeer's PR review:

The different segmentation methods will likely perform better with different hyperparameter configuration for each method. However it's difficult to be able to optimise these without calculating some evaluation metrics for the topic modelling. I'd suggest something like this:

def evaluation_metrics(topic_model, sentences, embeddings):
    topics, probs = topic_model.transform(sentences)
    
    # 1. Semantic Coherence
    def calculate_semantic_coherence(topic_sentences, topic_embeddings):
        if len(topic_sentences) < 2:
            return 0
        similarities = cosine_similarity(topic_embeddings)
        return np.mean(similarities)
    
    coherence_scores = {}
    for topic in set(topics):
        if topic != -1:  # Exclude outlier topic
            topic_mask = topics == topic
            topic_sentences = [sentences[i] for i in range(len(sentences)) if topic_mask[i]]
            topic_embeddings = embeddings[topic_mask]
            coherence_scores[topic] = calculate_semantic_coherence(topic_sentences, topic_embeddings)
    avg_coherence = np.mean(list(coherence_scores.values()))
    
    # 2. Topic Diversity
    def calculate_topic_diversity(topic_embeddings):
        centroid_embeddings = [np.mean(emb, axis=0) for emb in topic_embeddings if len(emb) > 0]
        if len(centroid_embeddings) < 2:
            return 0
        similarities = cosine_similarity(centroid_embeddings)
        return 1 - np.mean(similarities[np.triu_indices(len(similarities), k=1)])
    
    topic_embeddings = [embeddings[topics == topic] for topic in set(topics) if topic != -1]
    topic_diversity = calculate_topic_diversity(topic_embeddings)
    
    # 3. Topic Sizes
    topic_sizes = topic_model.get_topic_info().set_index("Name")["Count"]
    
    # 4. Document-Topic Representation
    avg_prob = np.mean(np.max(probs, axis=1))
    
    # 5. Silhouette Score
    silhouette_avg = silhouette_score(embeddings, topics)
    
    return {
        "average_semantic_coherence": avg_coherence,
        "topic_diversity": topic_diversity,
        "top_5_topics": topic_sizes[:5].to_dict(),
        "bottom_5_topics": topic_sizes[-5:].to_dict(),
        "average_max_topic_probability": avg_prob,
        "silhouette_score": silhouette_avg
    }

Using these metrics, you could perform a grid search or some Bayesian optimisation to improve the performance of the topic modelling.