marvinscham/masterthesis-playground
1
0
Fork 0
mirror of https://github.com/marvinscham/masterthesis-playground.git synced 2026-03-23 00:42:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

BERTopic cleanup and structuring

Browse Source
This commit is contained in:
Marvin Scham
2026-02-20 18:01:46 +01:00
parent 99ba5031ca
commit ccf96b447c
7 changed files with 55743 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
bertopic/bertopic_autotune.py
View File

@@ -8,7 +8,6 @@ from bertopic.vectorizers import ClassTfidfTransformer
from hdbscan import HDBSCAN
from sentence_transformers import SentenceTransformer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics import pairwise_distances
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.model_selection import ParameterGrid
from umap import UMAP
@@ -74,7 +73,7 @@ def auto_tune_bertopic(texts, embedding_model, param_grid):
print("Running embedding model...")
embedder = SentenceTransformer(embedding_model)
embeddings = embedder.encode(reviews, show_progress_bar=True)
embeddings = embedder.encode(texts, show_progress_bar=True)
# Convert param_grid to list for sampling
print("Generating parameter combinations...")
@@ -151,7 +150,9 @@ SPECIAL_CHARS = ["\n", "\\n"]
MIN_REVIEW_WORDS = 5
print("Loading reviews...")
reviews = pd.read_csv("../data/original/reviews.tab", sep="\t").review.to_list()
reviews = pd.read_csv(
"../data/intermediate/preprocessed.tab", sep="\t"
).review.to_list()
print("Running light preprocessing...")
for schar in SPECIAL_CHARS: