Fix script paths & remove redundant task

src/gurubase-backend/.devcontainer/devcontainer.json

@@ -4,7 +4,7 @@
 	"dockerFile": "../Dockerfile.dev",
 	"runArgs": [
 		"--name",
-		"kubernetesguru-backend",
+		"gurubase-backend-dev",
 		"-v",
 		"${env:HOME}${env:USERPROFILE}/.ssh:/root/.ssh-localhost:ro",
 		"--ipc",
@@ -13,7 +13,7 @@
 		// "gurubase-dc",
 		"anteon",
 		"--hostname",
-		"kubernetesguru-backend",
+		"gurubase-backend-dev",
 		"--env-file",
 		"envs/dev.env",
 		"-p",

src/gurubase-backend/backend/core/tasks.py

@@ -15,7 +15,6 @@
 from core.guru_types import get_guru_type_names, get_guru_type_object, get_guru_types_dict
 from core.models import DataSource, Favicon, GuruType, LLMEval, LinkReference, LinkValidity, Question, RawQuestion, RawQuestionGeneration, Summarization, SummaryQuestionGeneration, LLMEvalResult, GuruType, GithubFile
 from core.utils import finalize_data_source_summarizations, embed_texts, generate_questions_from_summary, generate_similar_questions, get_links, get_llm_usage, get_milvus_client, get_more_seo_friendly_title, get_most_similar_questions, guru_type_has_enough_generated_questions, create_guru_type_summarization, simulate_summary_and_answer, validate_guru_type, vector_db_fetch, with_redis_lock, generate_og_image, parse_context_from_prompt, get_default_settings, send_question_request_for_cloudflare_cache, send_guru_type_request_for_cloudflare_cache
-from core.stackoverflow_utils import get_most_popular_questions
 from django.conf import settings
 import time
 import re
@@ -204,71 +203,6 @@ def find_duplicate_question_titles():
             logger.fatal(f"Question has duplicate title: {question.question}. ID: {question.id}")
     logger.info('Found duplicate question titles')
 
-@shared_task
-def generate_raw_questions(guru_type, sort, page_num, page_size, generate_count, model):
-    logger.info(f'Generating raw questions for {guru_type} with {model}')
-    try:
-        validate_guru_type(guru_type)
-    except Exception:
-        logger.error(f"Invalid guru type: {guru_type}")
-        return
-
-    valid_sorts = ['activity', 'votes', 'creation', 'hot', 'week', 'month']
-
-    if sort not in valid_sorts:
-        logger.error(f"Invalid sort: {sort}. Valid values are: {valid_sorts}")
-        return
-
-    if page_num <= 0:
-        logger.error(f"Invalid page number: {page_num}")
-        return
-
-    valid_models = ['claude', 'chatgpt', 'gemini']
-    if model not in valid_models:
-        logger.error(f"Invalid model: {model}. Valid values are: {valid_models}")
-        return
-
-    guru_type_object = get_guru_type_object(guru_type)
-
-    # last week
-    from_date = int((datetime.now() - relativedelta(weeks=1)).timestamp())
-    to_date = int((datetime.now()).timestamp())
-    popular_questions = get_most_popular_questions(guru_type, sort, page_num, page_size, from_date, to_date)
-
-    raw_question_generation = RawQuestionGeneration.objects.create(guru_type=guru_type_object, sort=sort, page_num=page_num, page_size=page_size, generate_count=0, model=model)
-    total_completion_tokens = 0
-    total_prompt_tokens = 0
-    total_cached_prompt_tokens = 0
-    while generate_count > 0:
-        try:
-            similar_questions, completion_tokens, prompt_tokens, cached_prompt_tokens, prompts = generate_similar_questions(model, guru_type, popular_questions, generate_count)
-            total_completion_tokens += completion_tokens
-            total_prompt_tokens += prompt_tokens
-            total_cached_prompt_tokens += cached_prompt_tokens
-            raw_question_generation.prompts.extend(prompts)
-        except Exception as e:
-            logger.error(f"Error while generating similar questions: {traceback.format_exc()}")
-            return
-
-        c = 0
-        for category, questions in similar_questions.items():
-            for q in questions:
-                RawQuestion.objects.create(
-                    question=q,
-                    category=category,
-                    guru_type=guru_type_object,
-                    raw_question_generation=raw_question_generation,
-                )
-                c += 1 
-
-        raw_question_generation.generate_count += c 
-        generate_count -= c
-
-    cost_dollars = get_llm_usage(model, total_prompt_tokens, total_completion_tokens, total_cached_prompt_tokens)
-    raw_question_generation.cost_dollars = cost_dollars
-    raw_question_generation.save()
-    logger.info(f'generate_raw_questions for {guru_type} with {model} successfully completed, RawQuestionGeneration id:{raw_question_generation.id}')
-
 
 @shared_task
 @with_redis_lock(

src/gurubase-backend/backend/scripts/1_generate_users.py

@@ -3,7 +3,7 @@
 import sys
 
 import django
-sys.path.append('/workspaces/kubernetesguru-backend/backend')
+sys.path.append('/workspaces/gurubase/src/gurubase-backend/backend')
 sys.path.append('/workspaces/gurubase-backend/backend')
 sys.path.append('/workspace/backend')
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings")

src/gurubase-backend/backend/scripts/2_generate_periodic_tasks.py

@@ -5,7 +5,7 @@
 
 import django
 
-sys.path.append('/workspaces/kubernetesguru-backend/backend')
+sys.path.append('/workspaces/gurubase/src/gurubase-backend/backend')
 sys.path.append('/workspaces/gurubase-backend/backend')
 sys.path.append('/workspace/backend')
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings")

src/gurubase-backend/backend/scripts/3_update_site.py

@@ -3,7 +3,7 @@
 import sys
 
 import django
-sys.path.append('/workspaces/kubernetesguru-backend/backend')
+sys.path.append('/workspaces/gurubase/src/gurubase-backend/backend')
 sys.path.append('/workspaces/gurubase-backend/backend')
 sys.path.append('/workspace/backend')
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings")

src/gurubase-backend/backend/scripts/4_create_milvus_collections.py

@@ -3,7 +3,7 @@
 import sys
 
 import django
-sys.path.append('/workspaces/kubernetesguru-backend/backend')
+sys.path.append('/workspaces/gurubase/src/gurubase-backend/backend')
 sys.path.append('/workspaces/gurubase-backend/backend')
 sys.path.append('/workspace/backend')
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings")

src/gurubase-backend/backend/scripts/5_fill_llm_prices.py

@@ -6,7 +6,7 @@
 
 sys.path.append('/workspaces/gurubase-backend/backend')
 sys.path.append('/workspace/backend')
-sys.path.append('/workspaces/kubernetesguru-backend/backend')
+sys.path.append('/workspaces/gurubase/src/gurubase-backend/backend')
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings")
 
 django.setup()