bertopic.py

# -*- coding: utf-8 -*-
"""BERTopic.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1FieRA9fLdkQEGDIMYl0I3MCjSUKVF8C-

# **Tutorial** - Topic Modeling with BERTopic
(last updated 08-06-2021)

In this tutorial we will be exploring how to use BERTopic to create topics from the well-known 20Newsgroups dataset. The most frequent use-cases and methods are discussed together with important parameters to keep a look out for. 


## BERTopic
BERTopic is a topic modeling technique that leverages 🤗 transformers and a custom class-based TF-IDF to create dense clusters allowing for easily interpretable topics whilst keeping important words in the topic descriptions. 

<br>

<img src="https://raw.githubusercontent.com/MaartenGr/BERTopic/master/images/logo.png" width="40%">

# Enabling the GPU

First, you'll need to enable GPUs for the notebook:

- Navigate to Edit→Notebook Settings
- select GPU from the Hardware Accelerator drop-down

[Reference](https://colab.research.google.com/notebooks/gpu.ipynb)

# **Installing BERTopic**

We start by installing BERTopic from PyPi:
"""

# Commented out IPython magic to ensure Python compatibility.
# %%capture
# !pip install bertopic

"""## Restart the Notebook
After installing BERTopic, some packages that were already loaded were updated and in order to correctly use them, we should now restart the notebook.

From the Menu:

Runtime → Restart Runtime

# Data
For this example, we use the popular 20 Newsgroups dataset which contains roughly 18000 newsgroups posts
"""

#from sklearn.datasets import fetch_20newsgroups
#docs = fetch_20newsgroups(subset='all',  remove=('headers', 'footers', 'quotes'))['data']
#print(docs[0])
#print(type(docs[0]))
#print(len(docs))

import csv
import string 
from nltk.corpus import stopwords
import nltk
import pandas as pd
from pandas import DataFrame
import spacy
import re
import contractions
import time

print('Loading spacy')
nlp = spacy.load('en_core_web_sm')

def remove_punctuation(text):
    no_punct=[words for words in text if words not in string.punctuation]
    words_wo_punct=''.join(no_punct)
    return words_wo_punct

def expand_contractions(text):
    return contractions.fix(text)

def remove_special_chars(text, remove_digits=False):
    pattern = r'[^a-zA-Z0-9\s]' if not remove_digits else r'[^a-zA-Z\s]'
    return re.sub(pattern, '', text)

def remove_stopwords(text):
    stopword_list = stopwords.words('english')
    tokens = nltk.word_tokenize(text)
    tokens = [token.strip() for token in tokens]
    return ' '.join([token for token in tokens if token not in stopword_list])

def lemmatize(text):
    text = nlp(text)
    return ' '.join([word.lemma_ if word.lemma_ != '-PRON-' else word.text for word in text])

#with open('AIFORWOMEN_2022_RB_cl.csv', newline='') as f:
#with open('AGGREGATED_20-21_cl.csv', newline='') as f:
with open('AGGREGATED_20-21-22_cl_v2.csv', newline='') as f:    
    reader = csv.reader(f)
    docs_raw = list(reader)


docs=[]
count=0
print("Cleaning data...")
for line in docs_raw:
    l=remove_punctuation(line)
    l2=expand_contractions(l)
    l3=lemmatize(l2)
    l4=remove_special_chars(l3)
    l5=remove_stopwords(l4)
    docs.append(l5) 
    count+=1
print(f'input has {count} lines')

#print(data[0])
#print(type(data[0]))
"""# **Topic Modeling**

In this example, we will go through the main components of BERTopic and the steps necessary to create a strong topic model.

## Training

#We start by instantiating BERTopic. We set language to `english` since our documents are in the English language. If you would like to use a multi-lingual model, please use `language="multilingual"` instead. 

We will also calculate the topic probabilities. However, this can slow down BERTopic significantly at large amounts of data (>100_000 documents). It is advised to turn this off if you want to speed up the model.
"""

from bertopic import BERTopic
from sentence_transformers import SentenceTransformer

# Create embeddings
sentence_model_name = "all-MiniLM-L6-v2" #all-MiniLM-L12-v2 #"all-mpnet-base-v2"
sentence_model = SentenceTransformer(sentence_model_name)
#(https://www.sbert.net/docs/pretrained_models.html) for a list of supported sentence transformers models.  
#sentence_model = SentenceTransformer("all-mpnet-base-v2") all-MiniLM-L12-v2
#sentence_model = SentenceTransformer("all-MiniLM-L12-v2")
embeddings = sentence_model.encode(docs, show_progress_bar=True)

out_file_name=sentence_model_name + '_20-21-22-Data_' + time.strftime("%Y%m%d-%H%M%S")
excel_writer = pd.ExcelWriter("out_"+out_file_name + '.xlsx')


# Create topic model
topic_model = BERTopic(language="english", calculate_probabilities=True, verbose=True)
topics, probs = topic_model.fit_transform(docs, embeddings)
res=pd.concat([pd.DataFrame(docs_raw), pd.DataFrame(docs), pd.DataFrame(topics), pd.DataFrame(probs)],axis=1)
res.to_excel(excel_writer, sheet_name="pred")

#topic_model = BERTopic(language="english", calculate_probabilities=True, verbose=True)
#topics, probs = topic_model.fit_transform(docs)

"""**NOTE**: Use `language="multilingual"` to select a model that support 50+ languages.

## Extracting Topics
After fitting our model, we can start by looking at the results. Typically, we look at the most frequent topics first as they best represent the collection of documents.
"""

freq = topic_model.get_topic_info(); #freq.head(5)
#print(freq)

"""-1 refers to all outliers and should typically be ignored. Next, let's take a look at a frequent topic that were generated:"""

#topic_model.get_topic(2)  # Select the most frequent topic

"""# **Topic Representation**
After having created the topic model, you might not be satisfied with some of the parameters you have chosen. Fortunately, BERTopic allows you to update the topics after they have been created. 

This allows for fine-tuning the model to your specifications and wishes.

## Update Topics
When you have trained a model and viewed the topics and the words that represent them,
you might not be satisfied with the representation. Perhaps you forgot to remove
stopwords or you want to try out a different `n_gram_range`. We can use the function `update_topics` to update 
the topic representation with new parameters for `c-TF-IDF`:
"""

#topic_model.update_topics(docs, topics, n_gram_range=(1, 2))

from sklearn.feature_extraction.text import CountVectorizer
vectorizer_model = CountVectorizer(analyzer="word", ngram_range=(1, 2), stop_words=["always","never","like","someone","woman","make","want","good","way","take","let","see","thing"])
topic_model.update_topics(docs, topics, vectorizer_model=vectorizer_model)

topic_model.get_topic(0)   # We select topic that we viewed before

fig = topic_model.visualize_topics()
fig.write_html("visualized_topics" +out_file_name + ".html")
freq = topic_model.get_topic_info(); #freq.head(5)
print(freq) 
freq.to_excel(excel_writer, sheet_name="freq")
excel_writer.save()

"""**NOTE**: BERTopic is stocastich which mmeans that the topics might differ across runs. This is mostly due to the stocastisch nature of UMAP.

# **Visualization**
There are several visualization options available in BERTopic, namely the visualization of topics, probabilities and topics over time. Topic modeling is, to a certain extent, quite subjective. Visualizations help understand the topics that were created.

## Visualize Topics
After having trained our `BERTopic` model, we can iteratively go through perhaps a hundred topic to get a good 
understanding of the topics that were extract. However, that takes quite some time and lacks a global representation. 
Instead, we can visualize the topics that were generated in a way very similar to 
[LDAvis](https://github.com/cpsievert/LDAvis):
"""


"""## Visualize Topic Probabilities

The variable `probabilities` that is returned from `transform()` or `fit_transform()` can 
be used to understand how confident BERTopic is that certain topics can be found in a document. 

To visualize the distributions, we simply call:
"""

#topic_model.visualize_distribution(probs[200], min_probability=0.015)
fig = topic_model.visualize_distribution(probs[200], min_probability=0.015)
fig.write_html("distribution" +out_file_name + ".html")

"""## Visualize Topic Hierarchy

The topics that were created can be hierarchically reduced. In order to understand the potential hierarchical structure of the topics, we can use scipy.cluster.hierarchy to create clusters and visualize how they relate to one another. This might help selecting an appropriate nr_topics when reducing the number of topics that you have created.
"""

#topic_model.visualize_hierarchy(top_n_topics=50)
fig = topic_model.visualize_hierarchy(top_n_topics=50)
fig.write_html("hierarchy" +out_file_name + ".html")

"""## Visualize Terms

We can visualize the selected terms for a few topics by creating bar charts out of the c-TF-IDF scores for each topic representation. Insights can be gained from the relative c-TF-IDF scores between and within topics. Moreover, you can easily compare topic representations to each other.
"""

#topic_model.visualize_barchart(top_n_topics=5)
fig = topic_model.visualize_barchart(top_n_topics=5)
fig.write_html("barchart" +out_file_name + ".html")

"""## Visualize Topic Similarity
Having generated topic embeddings, through both c-TF-IDF and embeddings, we can create a similarity matrix by simply applying cosine similarities through those topic embeddings. The result will be a matrix indicating how similar certain topics are to each other.
"""

#topic_model.visualize_heatmap(n_clusters=20, width=1000, height=1000)
#fig = topic_model.visualize_heatmap(n_clusters=5, width=1000, height=1000)
#fig.write_html("heatmap.html")

"""## Visualize Term Score Decline
Topics are represented by a number of words starting with the best representative word. Each word is represented by a c-TF-IDF score. The higher the score, the more representative a word to the topic is. Since the topic words are sorted by their c-TF-IDF score, the scores slowly decline with each word that is added. At some point adding words to the topic representation only marginally increases the total c-TF-IDF score and would not be beneficial for its representation.

To visualize this effect, we can plot the c-TF-IDF scores for each topic by the term rank of each word. In other words, the position of the words (term rank), where the words with the highest c-TF-IDF score will have a rank of 1, will be put on the x-axis. Whereas the y-axis will be populated by the c-TF-IDF scores. The result is a visualization that shows you the decline of c-TF-IDF score when adding words to the topic representation. It allows you, using the elbow method, the select the best number of words in a topic.

"""

#topic_model.visualize_term_rank()
fig = topic_model.visualize_term_rank()
fig.write_html("term_rank" +out_file_name + ".html")



"""# **Model serialization**
The model and its internal settings can easily be saved. Note that the documents and embeddings will not be saved. However, UMAP and HDBSCAN will be saved. 
"""

# Save model
topic_model.save("model_"+out_file_name) #"my_model")


# Load model
#my_model = BERTopic.load("my_model")


"""## Topic Reduction
We can also reduce the number of topics after having trained a BERTopic model. The advantage of doing so, 
is that you can decide the number of topics after knowing how many are actually created. It is difficult to 
predict before training your model how many topics that are in your documents and how many will be extracted. 
Instead, we can decide afterwards how many topics seems realistic:
"""

#this is just keeping the top nr clusters and increasing the size of the [-1] topic
#new_topics, new_probs = topic_model.reduce_topics(docs, topics, probs, nr_topics=6)
#res=pd.concat([pd.DataFrame(docs_raw), pd.DataFrame(docs), pd.DataFrame(new_topics), pd.DataFrame(new_probs)],axis=1)
#res.to_excel(excel_writer, sheet_name="pred_fewer")
#freq = topic_model.get_topic_info(); #freq.head(5)
#print(freq) 
#freq.to_excel(excel_writer, sheet_name="freq_fewer")
#excel_writer.save()

"""# **Search Topics**
After having trained our model, we can use `find_topics` to search for topics that are similar 
to an input search_term. Here, we are going to be searching for topics that closely relate the 
search term "vehicle". Then, we extract the most similar topic and check the results: 
"""

#similar_topics, similarity = topic_model.find_topics("collaboration", top_n=5); similar_topics

#topic_model.get_topic(71)


"""# **Embedding Models**
The parameter `embedding_model` takes in a string pointing to a sentence-transformers model, a SentenceTransformer, or a Flair DocumentEmbedding model.

## Sentence-Transformers
You can select any model from sentence-transformers here and pass it through BERTopic with embedding_model:
"""

topic_model = BERTopic(embedding_model="xlm-r-bert-base-nli-stsb-mean-tokens")

"""Or select a SentenceTransformer model with your own parameters:

"""

from sentence_transformers import SentenceTransformer

sentence_model = SentenceTransformer("distilbert-base-nli-mean-tokens", device="cpu")
topic_model = BERTopic(embedding_model=sentence_model, verbose=True)

"""Click [here](https://www.sbert.net/docs/pretrained_models.html) for a list of supported sentence transformers models.  

"""