radames/sentence-embeddings-visualization

1from umap_reducer import UMAPReducer2from embeddings_encoder import EmbeddingsEncoder3from flask import Flask, request, render_template, jsonify, make_response, session4from flask_session import Session5from flask_cors import CORS, cross_origin6import os7from dotenv import load_dotenv8import feedparser9import json10from dateutil import parser11import re12import numpy as np13import gzip14import hashlib15 16load_dotenv()17 18 19app = Flask(__name__, static_url_path='/static')20app.config["SECRET_KEY"] = os.environ.get("SECRET_KEY") 21app.config["SESSION_PERMANENT"] = True22app.config["SESSION_TYPE"] = "filesystem"23app.config["SESSION_COOKIE_SAMESITE"] = "None"24app.config["SESSION_COOKIE_SECURE"] = True25Session(app)26CORS(app)27 28reducer = UMAPReducer()29encoder = EmbeddingsEncoder()30 31 32@app.route('/')33def index():34    return render_template('index.html')35 36 37@app.route('/run-umap', methods=['POST'])38@cross_origin(supports_credentials=True)39def run_umap():40    input_data = request.get_json()41    sentences = input_data['data']['sentences']42    umap_options = input_data['data']['umap_options']43    cluster_options = input_data['data']['cluster_options']44    # create unique hash for input, avoid recalculating embeddings45    sentences_input_hash = hashlib.sha256(46        ''.join(sentences).encode("utf-8")).hexdigest()47 48    print("input options:", sentences_input_hash,49          umap_options, cluster_options, "\n\n")50    try:51        if not session.get(sentences_input_hash):52            print("New input, calculating embeddings" "\n\n")53            embeddings = encoder.encode(sentences)54            session[sentences_input_hash] = embeddings.tolist()55        else:56            print("Input already calculated, using cached embeddings", "\n\n")57            embeddings = session[sentences_input_hash]58 59        # UMAP embeddings60        reducer.setParams(umap_options, cluster_options)61        umap_embeddings = reducer.embed(embeddings)62        # HDBScan cluster analysis63        clusters = reducer.clusterAnalysis(umap_embeddings)64        content = gzip.compress(json.dumps(65            {66                "embeddings": umap_embeddings.tolist(),67                "clusters": clusters.labels_.tolist()68            }69        ).encode('utf8'), 5)70        response = make_response(content)71        response.headers['Content-length'] = len(content)72        response.headers['Content-Encoding'] = 'gzip'73        return response74    except Exception as e:75        return jsonify({"error": str(e)}), 40076 77 78if __name__ == '__main__':79    app.run(host='0.0.0.0',  port=int(os.environ.get('PORT', 7860)))80