basant307/AI_Governance_Project
048
1/**2 * @license3 * Copyright 2025 Qwen Code4 * SPDX-License-Identifier: Apache-2.05 */6 7import fs from 'fs/promises';8import path from 'path';9import {10 read as readJsonlFile,11 createDebugLogger,12} from '@qwen-code/qwen-code-core';13import pLimit from 'p-limit';14import type {15 InsightData,16 HeatMapData,17 StreakData,18 SessionFacets,19 InsightProgressCallback,20} from '../types/StaticInsightTypes.js';21import type {22 QualitativeInsights,23 InsightImpressiveWorkflows,24 InsightProjectAreas,25 InsightFutureOpportunities,26 InsightFrictionPoints,27 InsightMemorableMoment,28 InsightImprovements,29 InsightInteractionStyle,30 InsightAtAGlance,31} from '../types/QualitativeInsightTypes.js';32import {33 getInsightPrompt,34 runSideQuery,35 type Config,36 type ChatRecord,37} from '@qwen-code/qwen-code-core';38 39const logger = createDebugLogger('DataProcessor');40 41const CONCURRENCY_LIMIT = 4;42const SESSION_OUTCOMES = [43 'fully_achieved',44 'mostly_achieved',45 'partially_achieved',46 'not_achieved',47 'unclear_from_transcript',48] as const;49const OUTCOME_FALLBACK = 'unclear_from_transcript';50const QWEN_HELPFULNESS_LEVELS = [51 'unhelpful',52 'slightly_helpful',53 'moderately_helpful',54 'very_helpful',55 'essential',56] as const;57const SESSION_TYPES = [58 'single_task',59 'multi_task',60 'iterative_refinement',61 'exploration',62 'quick_question',63] as const;64const PRIMARY_SUCCESS_VALUES = [65 'none',66 'fast_accurate_search',67 'correct_code_edits',68 'good_explanations',69 'proactive_help',70 'multi_file_changes',71 'good_debugging',72] as const;73const PRIMARY_SUCCESS_FALLBACK = 'none';74 75// Keep in sync with packages/web-templates/src/insight/src/App.tsx.76function hasMeaningfulInsightValue(value: unknown): boolean {77 if (typeof value === 'string') {78 return value.trim().length > 0;79 }80 81 if (typeof value === 'number') {82 return Number.isFinite(value) && value !== 0;83 }84 85 if (typeof value === 'boolean') {86 return value;87 }88 89 if (Array.isArray(value)) {90 return value.some((item) => hasMeaningfulInsightValue(item));91 }92 93 if (value && typeof value === 'object') {94 return Object.values(value).some((item) => hasMeaningfulInsightValue(item));95 }96 97 return false;98}99 100function normalizeInsightText(value: unknown): string {101 return typeof value === 'string' ? value.trim() : '';102}103 104function normalizeInsightCountRecord(value: unknown): Record<string, number> {105 if (!value || typeof value !== 'object' || Array.isArray(value)) {106 return {};107 }108 109 return Object.entries(value).reduce<Record<string, number>>(110 (acc, [key, count]) => {111 if (typeof count === 'number' && Number.isFinite(count) && count > 0) {112 acc[key] = count;113 }114 return acc;115 },116 {},117 );118}119 120function getInsightCountEntries(value: unknown): Array<[string, number]> {121 return Object.entries(normalizeInsightCountRecord(value));122}123 124function normalizeInsightEnum<T extends string>(125 value: unknown,126 allowed: readonly T[],127 fallback: T,128): T {129 const trimmed = typeof value === 'string' ? value.trim() : '';130 if (trimmed) {131 const match = allowed.find(132 (item) => String(item).toLowerCase() === trimmed.toLowerCase(),133 );134 if (match) return match;135 }136 137 logger.debug(138 `Normalized unknown insight enum value "${String(value)}" to fallback "${fallback}"`,139 );140 return fallback;141}142 143function normalizeSessionFacet(144 facet: unknown,145 sessionId: string,146): SessionFacets | null {147 if (!facet || typeof facet !== 'object' || Array.isArray(facet)) {148 return null;149 }150 151 const rawFacet = facet as Record<string, unknown>;152 const normalizedFacet: SessionFacets = {153 session_id: sessionId,154 underlying_goal: normalizeInsightText(rawFacet['underlying_goal']),155 goal_categories: normalizeInsightCountRecord(rawFacet['goal_categories']),156 outcome: normalizeInsightEnum(157 rawFacet['outcome'],158 SESSION_OUTCOMES,159 OUTCOME_FALLBACK,160 ),161 user_satisfaction_counts: normalizeInsightCountRecord(162 rawFacet['user_satisfaction_counts'],163 ),164 Qwen_helpfulness: normalizeInsightEnum(165 rawFacet['Qwen_helpfulness'],166 QWEN_HELPFULNESS_LEVELS,167 'moderately_helpful',168 ),169 session_type: normalizeInsightEnum(170 rawFacet['session_type'],171 SESSION_TYPES,172 'single_task',173 ),174 friction_counts: normalizeInsightCountRecord(rawFacet['friction_counts']),175 friction_detail: normalizeInsightText(rawFacet['friction_detail']),176 primary_success: normalizeInsightEnum(177 rawFacet['primary_success'],178 PRIMARY_SUCCESS_VALUES,179 PRIMARY_SUCCESS_FALLBACK,180 ),181 brief_summary: normalizeInsightText(rawFacet['brief_summary']),182 };183 184 const meaningfulContent = {185 underlying_goal: normalizedFacet.underlying_goal,186 goal_categories: normalizedFacet.goal_categories,187 outcome:188 normalizedFacet.outcome === OUTCOME_FALLBACK189 ? ''190 : normalizedFacet.outcome,191 user_satisfaction_counts: normalizedFacet.user_satisfaction_counts,192 friction_counts: normalizedFacet.friction_counts,193 friction_detail: normalizedFacet.friction_detail,194 primary_success:195 normalizedFacet.primary_success === PRIMARY_SUCCESS_FALLBACK196 ? ''197 : normalizedFacet.primary_success,198 brief_summary: normalizedFacet.brief_summary,199 };200 201 return hasMeaningfulInsightValue(meaningfulContent) ? normalizedFacet : null;202}203 204export class DataProcessor {205 constructor(private config: Config) {}206 207 // Helper function to format date as YYYY-MM-DD208 private formatDate(date: Date): string {209 return date.toISOString().split('T')[0];210 }211 212 // Format chat records for LLM analysis213 private formatRecordsForAnalysis(records: ChatRecord[]): string {214 let output = '';215 const sessionStart =216 records.length > 0 ? new Date(records[0].timestamp) : new Date();217 218 output += `Session: ${records[0]?.sessionId || 'unknown'}\n`;219 output += `Date: ${sessionStart.toISOString()}\n`;220 output += `Duration: ${records.length} turns\n\n`;221 222 for (const record of records) {223 if (record.type === 'user') {224 const text =225 record.message?.parts226 ?.map((p) => ('text' in p ? p.text : ''))227 .join('') || '';228 output += `[User]: ${text}\n`;229 } else if (record.type === 'assistant') {230 if (record.message?.parts) {231 for (const part of record.message.parts) {232 if ('text' in part && part.text) {233 output += `[Assistant]: ${part.text}\n`;234 } else if ('functionCall' in part) {235 const call = part.functionCall;236 if (call) {237 output += `[Tool: ${call.name}]\n`;238 }239 }240 }241 }242 }243 }244 return output;245 }246 247 // Only analyze conversational sessions for facets (skip system-only logs).248 private hasUserAndAssistantRecords(records: ChatRecord[]): boolean {249 let hasUser = false;250 let hasAssistant = false;251 252 for (const record of records) {253 if (record.type === 'user') {254 hasUser = true;255 } else if (record.type === 'assistant') {256 hasAssistant = true;257 }258 259 if (hasUser && hasAssistant) {260 return true;261 }262 }263 264 return false;265 }266 267 // Analyze a single session using LLM268 private async analyzeSession(269 records: ChatRecord[],270 ): Promise<SessionFacets | null> {271 if (records.length === 0) return null;272 273 const INSIGHT_SCHEMA = {274 type: 'object',275 properties: {276 underlying_goal: {277 type: 'string',278 description: 'What the user fundamentally wanted to achieve',279 },280 goal_categories: {281 type: 'object',282 additionalProperties: { type: 'number' },283 },284 outcome: {285 type: 'string',286 enum: [287 'fully_achieved',288 'mostly_achieved',289 'partially_achieved',290 'not_achieved',291 'unclear_from_transcript',292 ],293 },294 user_satisfaction_counts: {295 type: 'object',296 additionalProperties: { type: 'number' },297 },298 Qwen_helpfulness: {299 type: 'string',300 enum: [301 'unhelpful',302 'slightly_helpful',303 'moderately_helpful',304 'very_helpful',305 'essential',306 ],307 },308 session_type: {309 type: 'string',310 enum: [311 'single_task',312 'multi_task',313 'iterative_refinement',314 'exploration',315 'quick_question',316 ],317 },318 friction_counts: {319 type: 'object',320 additionalProperties: { type: 'number' },321 },322 friction_detail: {323 type: 'string',324 description: 'One sentence describing friction or empty',325 },326 primary_success: {327 type: 'string',328 enum: [329 'none',330 'fast_accurate_search',331 'correct_code_edits',332 'good_explanations',333 'proactive_help',334 'multi_file_changes',335 'good_debugging',336 ],337 },338 brief_summary: {339 type: 'string',340 description: 'One sentence: what user wanted and whether they got it',341 },342 },343 required: [344 'underlying_goal',345 'goal_categories',346 'outcome',347 'user_satisfaction_counts',348 'Qwen_helpfulness',349 'session_type',350 'friction_counts',351 'friction_detail',352 'primary_success',353 'brief_summary',354 ],355 };356 357 const sessionText = this.formatRecordsForAnalysis(records);358 const prompt = `${getInsightPrompt('analysis')}\n\nSESSION:\n${sessionText}`;359 360 try {361 const result = await runSideQuery<Record<string, unknown>>(this.config, {362 purpose: 'insight-session-analysis',363 // Quality is the entire point — keep main model + reasoning on.364 model: this.config.getModel(),365 contents: [{ role: 'user', parts: [{ text: prompt }] }],366 schema: INSIGHT_SCHEMA,367 config: {368 thinkingConfig: { includeThoughts: true },369 },370 abortSignal: AbortSignal.timeout(600000), // 10 minute timeout per session371 });372 373 if (!result || Object.keys(result).length === 0) {374 return null;375 }376 377 const sessionId = records[0].sessionId;378 const normalizedFacet = normalizeSessionFacet(result, sessionId);379 380 if (!normalizedFacet) {381 logger.warn(382 `Ignoring malformed insight facet for session ${sessionId}`,383 );384 return null;385 }386 387 return normalizedFacet;388 } catch (error) {389 logger.error(390 `Failed to analyze session ${records[0]?.sessionId}:`,391 error,392 );393 return null;394 }395 }396 397 // Calculate streaks from activity dates398 private calculateStreaks(dates: string[]): StreakData {399 if (dates.length === 0) {400 return { currentStreak: 0, longestStreak: 0, dates: [] };401 }402 403 // Convert string dates to Date objects and sort them404 const dateObjects = dates.map((dateStr) => new Date(dateStr));405 dateObjects.sort((a, b) => a.getTime() - b.getTime());406 407 let currentStreak = 1;408 let maxStreak = 1;409 let currentDate = new Date(dateObjects[0]);410 currentDate.setHours(0, 0, 0, 0); // Normalize to start of day411 412 for (let i = 1; i < dateObjects.length; i++) {413 const nextDate = new Date(dateObjects[i]);414 nextDate.setHours(0, 0, 0, 0); // Normalize to start of day415 416 // Calculate difference in days417 const diffDays = Math.floor(418 (nextDate.getTime() - currentDate.getTime()) / (1000 * 60 * 60 * 24),419 );420 421 if (diffDays === 1) {422 // Consecutive day423 currentStreak++;424 maxStreak = Math.max(maxStreak, currentStreak);425 } else if (diffDays > 1) {426 // Gap in streak427 currentStreak = 1;428 }429 // If diffDays === 0, same day, so streak continues430 431 currentDate = nextDate;432 }433 434 // Check if the streak is still ongoing (if last activity was yesterday or today)435 const today = new Date();436 today.setHours(0, 0, 0, 0);437 const yesterday = new Date(today);438 yesterday.setDate(yesterday.getDate() - 1);439 440 if (441 currentDate.getTime() === today.getTime() ||442 currentDate.getTime() === yesterday.getTime()443 ) {444 // The streak might still be active, so we don't reset it445 }446 447 return {448 currentStreak,449 longestStreak: maxStreak,450 dates,451 };452 }453 454 // Process chat files from all projects in the base directory and generate insights455 async generateInsights(456 baseDir: string,457 facetsOutputDir?: string,458 onProgress?: InsightProgressCallback,459 ): Promise<InsightData> {460 if (onProgress) onProgress('Scanning chat history...', 0);461 const allChatFiles = await this.scanChatFiles(baseDir);462 463 if (onProgress) onProgress('Crunching the numbers', 10);464 const metrics = await this.generateMetrics(allChatFiles, onProgress);465 466 if (onProgress) onProgress('Preparing sessions...', 20);467 const facets = await this.generateFacets(468 allChatFiles,469 facetsOutputDir,470 onProgress,471 );472 473 if (onProgress) onProgress('Generating personalized insights...', 80);474 const qualitative = await this.generateQualitativeInsights(metrics, facets);475 476 // Aggregate satisfaction, friction, success and outcome data from facets477 const {478 satisfactionAgg,479 frictionAgg,480 primarySuccessAgg,481 outcomesAgg,482 goalsAgg,483 } = this.aggregateFacetsData(facets);484 485 if (onProgress) onProgress('Assembling report...', 100);486 487 return {488 ...metrics,489 qualitative,490 satisfaction: satisfactionAgg,491 friction: frictionAgg,492 primarySuccess: primarySuccessAgg,493 outcomes: outcomesAgg,494 topGoals: goalsAgg,495 };496 }497 498 // Aggregate satisfaction and friction data from facets499 private aggregateFacetsData(facets: SessionFacets[]): {500 satisfactionAgg: Record<string, number>;501 frictionAgg: Record<string, number>;502 primarySuccessAgg: Record<string, number>;503 outcomesAgg: Record<string, number>;504 goalsAgg: Record<string, number>;505 } {506 const satisfactionAgg: Record<string, number> = {};507 const frictionAgg: Record<string, number> = {};508 const primarySuccessAgg: Record<string, number> = {};509 const outcomesAgg: Record<string, number> = {};510 const goalsAgg: Record<string, number> = {};511 512 facets.forEach((facet) => {513 // Aggregate satisfaction514 getInsightCountEntries(facet.user_satisfaction_counts).forEach(515 ([sat, count]) => {516 satisfactionAgg[sat] = (satisfactionAgg[sat] || 0) + count;517 },518 );519 520 // Aggregate friction521 getInsightCountEntries(facet.friction_counts).forEach(([fric, count]) => {522 frictionAgg[fric] = (frictionAgg[fric] || 0) + count;523 });524 525 // Aggregate primary success526 const primarySuccess = normalizeInsightEnum(527 facet.primary_success,528 PRIMARY_SUCCESS_VALUES,529 PRIMARY_SUCCESS_FALLBACK,530 );531 if (primarySuccess !== PRIMARY_SUCCESS_FALLBACK) {532 primarySuccessAgg[primarySuccess] =533 (primarySuccessAgg[primarySuccess] || 0) + 1;534 }535 536 // Aggregate outcomes537 const outcome = normalizeInsightEnum(538 facet.outcome,539 SESSION_OUTCOMES,540 OUTCOME_FALLBACK,541 );542 outcomesAgg[outcome] = (outcomesAgg[outcome] || 0) + 1;543 544 // Aggregate goals545 getInsightCountEntries(facet.goal_categories).forEach(([goal, count]) => {546 goalsAgg[goal] = (goalsAgg[goal] || 0) + count;547 });548 });549 550 return {551 satisfactionAgg,552 frictionAgg,553 primarySuccessAgg,554 outcomesAgg,555 goalsAgg,556 };557 }558 559 private async generateQualitativeInsights(560 metrics: Omit<InsightData, 'facets' | 'qualitative'>,561 facets: SessionFacets[],562 ): Promise<QualitativeInsights | undefined> {563 if (facets.length === 0) {564 return undefined;565 }566 567 logger.info('Generating qualitative insights...');568 569 const commonData = this.prepareCommonPromptData(metrics, facets);570 571 const generate = async <T>(572 promptTemplate: string,573 schema: Record<string, unknown>,574 ): Promise<T | undefined> => {575 const prompt = `${promptTemplate}\n\n${commonData}`;576 try {577 const result = await runSideQuery<Record<string, unknown>>(578 this.config,579 {580 purpose: 'insight-qualitative-generate',581 model: this.config.getModel(),582 contents: [{ role: 'user', parts: [{ text: prompt }] }],583 schema,584 config: {585 thinkingConfig: { includeThoughts: true },586 },587 abortSignal: AbortSignal.timeout(600000),588 },589 );590 return result as T;591 } catch (error) {592 logger.error('Failed to generate insight:', error);593 return undefined;594 }595 };596 597 // Schemas for each insight type598 // We define simplified schemas here to guide the LLM.599 // The types are already defined in QualitativeInsightTypes.ts600 601 // 1. Impressive Workflows602 const schemaImpressiveWorkflows = {603 type: 'object',604 properties: {605 intro: { type: 'string' },606 impressive_workflows: {607 type: 'array',608 items: {609 type: 'object',610 properties: {611 title: { type: 'string' },612 description: { type: 'string' },613 },614 required: ['title', 'description'],615 },616 },617 },618 required: ['intro', 'impressive_workflows'],619 };620 621 // 2. Project Areas622 const schemaProjectAreas = {623 type: 'object',624 properties: {625 areas: {626 type: 'array',627 items: {628 type: 'object',629 properties: {630 name: { type: 'string' },631 session_count: { type: 'number' },632 description: { type: 'string' },633 },634 required: ['name', 'session_count', 'description'],635 },636 },637 },638 required: ['areas'],639 };640 641 // 3. Future Opportunities642 const schemaFutureOpportunities = {643 type: 'object',644 properties: {645 intro: { type: 'string' },646 opportunities: {647 type: 'array',648 items: {649 type: 'object',650 properties: {651 title: { type: 'string' },652 whats_possible: { type: 'string' },653 how_to_try: { type: 'string' },654 copyable_prompt: { type: 'string' },655 },656 required: [657 'title',658 'whats_possible',659 'how_to_try',660 'copyable_prompt',661 ],662 },663 },664 },665 required: ['intro', 'opportunities'],666 };667 668 // 4. Friction Points669 const schemaFrictionPoints = {670 type: 'object',671 properties: {672 intro: { type: 'string' },673 categories: {674 type: 'array',675 items: {676 type: 'object',677 properties: {678 category: { type: 'string' },679 description: { type: 'string' },680 examples: { type: 'array', items: { type: 'string' } },681 },682 required: ['category', 'description', 'examples'],683 },684 },685 },686 required: ['intro', 'categories'],687 };688 689 // 5. Memorable Moment690 const schemaMemorableMoment = {691 type: 'object',692 properties: {693 headline: { type: 'string' },694 detail: { type: 'string' },695 },696 required: ['headline', 'detail'],697 };698 699 // 6. Improvements700 const schemaImprovements = {701 type: 'object',702 properties: {703 Qwen_md_additions: {704 type: 'array',705 items: {706 type: 'object',707 properties: {708 addition: { type: 'string' },709 why: { type: 'string' },710 prompt_scaffold: { type: 'string' },711 },712 required: ['addition', 'why', 'prompt_scaffold'],713 },714 },715 features_to_try: {716 type: 'array',717 items: {718 type: 'object',719 properties: {720 feature: { type: 'string' },721 one_liner: { type: 'string' },722 why_for_you: { type: 'string' },723 example_code: { type: 'string' },724 },725 required: ['feature', 'one_liner', 'why_for_you', 'example_code'],726 },727 },728 usage_patterns: {729 type: 'array',730 items: {731 type: 'object',732 properties: {733 title: { type: 'string' },734 suggestion: { type: 'string' },735 detail: { type: 'string' },736 copyable_prompt: { type: 'string' },737 },738 required: ['title', 'suggestion', 'detail', 'copyable_prompt'],739 },740 },741 },742 required: ['Qwen_md_additions', 'features_to_try', 'usage_patterns'],743 };744 745 // 7. Interaction Style746 const schemaInteractionStyle = {747 type: 'object',748 properties: {749 narrative: { type: 'string' },750 key_pattern: { type: 'string' },751 },752 required: ['narrative', 'key_pattern'],753 };754 755 // 8. At A Glance756 const schemaAtAGlance = {757 type: 'object',758 properties: {759 whats_working: { type: 'string' },760 whats_hindering: { type: 'string' },761 quick_wins: { type: 'string' },762 ambitious_workflows: { type: 'string' },763 },764 required: [765 'whats_working',766 'whats_hindering',767 'quick_wins',768 'ambitious_workflows',769 ],770 };771 772 const limit = pLimit(CONCURRENCY_LIMIT);773 774 try {775 const [776 impressiveWorkflows,777 projectAreas,778 futureOpportunities,779 frictionPoints,780 memorableMoment,781 improvements,782 interactionStyle,783 atAGlance,784 ] = await Promise.all([785 limit(() =>786 generate<InsightImpressiveWorkflows>(787 getInsightPrompt('impressive_workflows'),788 schemaImpressiveWorkflows,789 ),790 ),791 limit(() =>792 generate<InsightProjectAreas>(793 getInsightPrompt('project_areas'),794 schemaProjectAreas,795 ),796 ),797 limit(() =>798 generate<InsightFutureOpportunities>(799 getInsightPrompt('future_opportunities'),800 schemaFutureOpportunities,801 ),802 ),803 limit(() =>804 generate<InsightFrictionPoints>(805 getInsightPrompt('friction_points'),806 schemaFrictionPoints,807 ),808 ),809 limit(() =>810 generate<InsightMemorableMoment>(811 getInsightPrompt('memorable_moment'),812 schemaMemorableMoment,813 ),814 ),815 limit(() =>816 generate<InsightImprovements>(817 getInsightPrompt('improvements'),818 schemaImprovements,819 ),820 ),821 limit(() =>822 generate<InsightInteractionStyle>(823 getInsightPrompt('interaction_style'),824 schemaInteractionStyle,825 ),826 ),827 limit(() =>828 generate<InsightAtAGlance>(829 getInsightPrompt('at_a_glance'),830 schemaAtAGlance,831 ),832 ),833 ]);834 835 logger.debug(836 JSON.stringify(837 {838 impressiveWorkflows,839 projectAreas,840 futureOpportunities,841 frictionPoints,842 memorableMoment,843 improvements,844 interactionStyle,845 atAGlance,846 },847 null,848 2,849 ),850 );851 852 const qualitative = {853 impressiveWorkflows,854 projectAreas,855 futureOpportunities,856 frictionPoints,857 memorableMoment,858 improvements,859 interactionStyle,860 atAGlance,861 };862 863 return hasMeaningfulInsightValue(qualitative) ? qualitative : undefined;864 } catch (e) {865 logger.error('Error generating qualitative insights:', e);866 return undefined;867 }868 }869 870 private prepareCommonPromptData(871 metrics: Omit<InsightData, 'facets' | 'qualitative'>,872 facets: SessionFacets[],873 ): string {874 // 1. DATA section875 const goalsAgg: Record<string, number> = {};876 const outcomesAgg: Record<string, number> = {};877 const satisfactionAgg: Record<string, number> = {};878 const frictionAgg: Record<string, number> = {};879 const successAgg: Record<string, number> = {};880 881 facets.forEach((facet) => {882 // Aggregate goals883 getInsightCountEntries(facet.goal_categories).forEach(([goal, count]) => {884 goalsAgg[goal] = (goalsAgg[goal] || 0) + count;885 });886 887 // Aggregate outcomes888 const outcome = normalizeInsightEnum(889 facet.outcome,890 SESSION_OUTCOMES,891 OUTCOME_FALLBACK,892 );893 outcomesAgg[outcome] = (outcomesAgg[outcome] || 0) + 1;894 895 // Aggregate satisfaction896 getInsightCountEntries(facet.user_satisfaction_counts).forEach(897 ([sat, count]) => {898 satisfactionAgg[sat] = (satisfactionAgg[sat] || 0) + count;899 },900 );901 902 // Aggregate friction903 getInsightCountEntries(facet.friction_counts).forEach(([fric, count]) => {904 frictionAgg[fric] = (frictionAgg[fric] || 0) + count;905 });906 907 // Aggregate success (primary_success)908 const primarySuccess = normalizeInsightEnum(909 facet.primary_success,910 PRIMARY_SUCCESS_VALUES,911 PRIMARY_SUCCESS_FALLBACK,912 );913 if (primarySuccess !== PRIMARY_SUCCESS_FALLBACK) {914 successAgg[primarySuccess] = (successAgg[primarySuccess] || 0) + 1;915 }916 });917 918 const topGoals = Object.entries(goalsAgg)919 .sort((a, b) => b[1] - a[1])920 .slice(0, 8);921 922 const dataObj = {923 sessions: metrics.totalSessions || facets.length,924 analyzed: facets.length,925 date_range: {926 start: Object.keys(metrics.heatmap).sort()[0] || 'N/A',927 end: Object.keys(metrics.heatmap).sort().pop() || 'N/A',928 },929 messages: metrics.totalMessages || 0,930 hours: metrics.totalHours || 0,931 commits: 0, // Not tracked yet932 top_tools: metrics.topTools || [],933 top_goals: topGoals,934 outcomes: outcomesAgg,935 satisfaction: satisfactionAgg,936 friction: frictionAgg,937 success: successAgg,938 };939 940 // 2. SESSION SUMMARIES section941 const sessionSummaries = facets942 .map((f) => normalizeInsightText(f.brief_summary))943 .filter((summary) => summary.length > 0)944 .map((summary) => `- ${summary}`)945 .join('\n');946 947 // 3. FRICTION DETAILS section948 const frictionDetails = facets949 .map((f) => normalizeInsightText(f.friction_detail))950 .filter((detail) => detail.length > 0)951 .map((detail) => `- ${detail}`)952 .join('\n');953 954 return `DATA:955${JSON.stringify(dataObj, null, 2)}956 957SESSION SUMMARIES:958${sessionSummaries}959 960FRICTION DETAILS:961${frictionDetails}962 963USER INSTRUCTIONS TO Qwen:964None captured`;965 }966 967 private async scanChatFiles(968 baseDir: string,969 ): Promise<Array<{ path: string; mtime: number }>> {970 const allChatFiles: Array<{ path: string; mtime: number }> = [];971 972 try {973 // Get all project directories in the base directory974 const projectDirs = await fs.readdir(baseDir);975 976 // Process each project directory977 for (const projectDir of projectDirs) {978 const projectPath = path.join(baseDir, projectDir);979 const stats = await fs.stat(projectPath);980 981 // Only process if it's a directory982 if (stats.isDirectory()) {983 const chatsDir = path.join(projectPath, 'chats');984 985 try {986 // Get all chat files in the chats directory987 const files = await fs.readdir(chatsDir);988 const chatFiles = files.filter((file) => file.endsWith('.jsonl'));989 990 for (const file of chatFiles) {991 const filePath = path.join(chatsDir, file);992 993 // Get file stats for sorting by recency994 try {995 const fileStats = await fs.stat(filePath);996 allChatFiles.push({ path: filePath, mtime: fileStats.mtimeMs });997 } catch (e) {998 logger.error(`Failed to stat file ${filePath}:`, e);999 }1000 }1001 } catch (error) {1002 if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {1003 logger.error(1004 `Error reading chats directory for project ${projectDir}: ${error}`,1005 );1006 }1007 // Continue to next project if chats directory doesn't exist1008 continue;1009 }1010 }1011 }1012 } catch (error) {1013 if ((error as NodeJS.ErrnoException).code === 'ENOENT') {1014 // Base directory doesn't exist, return empty1015 logger.info(`Base directory does not exist: ${baseDir}`);1016 } else {1017 logger.error(`Error reading base directory: ${error}`);1018 }1019 }1020 1021 return allChatFiles;1022 }1023 1024 private async generateMetrics(1025 files: Array<{ path: string; mtime: number }>,1026 onProgress?: InsightProgressCallback,1027 ): Promise<Omit<InsightData, 'facets' | 'qualitative'>> {1028 // Initialize data structures1029 const heatmap: HeatMapData = {};1030 const activeHours: { [hour: number]: number } = {};1031 const sessionStartTimes: { [sessionId: string]: Date } = {};1032 const sessionEndTimes: { [sessionId: string]: Date } = {};1033 let totalMessages = 0;1034 let totalLinesAdded = 0;1035 let totalLinesRemoved = 0;1036 const uniqueFiles = new Set<string>();1037 const toolUsage: Record<string, number> = {};1038 1039 // Process files in batches to avoid OOM and blocking the event loop1040 const BATCH_SIZE = 50;1041 const totalFiles = files.length;1042 1043 for (let i = 0; i < totalFiles; i += BATCH_SIZE) {1044 const batchEnd = Math.min(i + BATCH_SIZE, totalFiles);1045 const batch = files.slice(i, batchEnd);1046 1047 // Process batch sequentially to minimize memory usage1048 for (const fileInfo of batch) {1049 try {1050 const records = await readJsonlFile<ChatRecord>(fileInfo.path);1051 1052 // Process each record1053 for (const record of records) {1054 const timestamp = new Date(record.timestamp);1055 const dateKey = this.formatDate(timestamp);1056 const hour = timestamp.getHours();1057 1058 // Count user messages and slash commands (actual user interactions)1059 const isUserMessage = record.type === 'user';1060 const isSlashCommand =1061 record.type === 'system' && record.subtype === 'slash_command';1062 if (isUserMessage || isSlashCommand) {1063 totalMessages++;1064 1065 // Update heatmap (count of user interactions per day)1066 heatmap[dateKey] = (heatmap[dateKey] || 0) + 1;1067 1068 // Update active hours1069 activeHours[hour] = (activeHours[hour] || 0) + 1;1070 }1071 1072 // Track session times1073 if (!sessionStartTimes[record.sessionId]) {1074 sessionStartTimes[record.sessionId] = timestamp;1075 }1076 sessionEndTimes[record.sessionId] = timestamp;1077 1078 // Track tool usage1079 if (record.type === 'assistant' && record.message?.parts) {1080 for (const part of record.message.parts) {1081 if ('functionCall' in part) {1082 const name = part.functionCall!.name!;1083 toolUsage[name] = (toolUsage[name] || 0) + 1;1084 }1085 }1086 }1087 1088 // Track lines and files from tool results1089 if (1090 record.type === 'tool_result' &&1091 record.toolCallResult?.resultDisplay1092 ) {1093 const display = record.toolCallResult.resultDisplay;1094 // Check if it matches FileDiff shape1095 if (1096 typeof display === 'object' &&1097 display !== null &&1098 'fileName' in display1099 ) {1100 // Cast to any to avoid importing FileDiff type which might not be available here1101 const diff = display as {1102 fileName: unknown;1103 diffStat?: {1104 model_added_lines?: number;1105 model_removed_lines?: number;1106 };1107 };1108 if (typeof diff.fileName === 'string') {1109 uniqueFiles.add(diff.fileName);1110 }1111 1112 if (diff.diffStat) {1113 totalLinesAdded += diff.diffStat.model_added_lines || 0;1114 totalLinesRemoved += diff.diffStat.model_removed_lines || 0;1115 }1116 }1117 }1118 }1119 } catch (error) {1120 logger.error(1121 `Failed to process metrics for file ${fileInfo.path}:`,1122 error,1123 );1124 // Continue to next file1125 }1126 }1127 1128 // Update progress (mapped to 10-20% range of total progress)1129 if (onProgress) {1130 const percentComplete = batchEnd / totalFiles;1131 const overallProgress = 10 + Math.round(percentComplete * 10);1132 onProgress(1133 `Crunching the numbers (${batchEnd}/${totalFiles})`,1134 overallProgress,1135 );1136 }1137 1138 // Yield to event loop to allow GC and UI updates1139 await new Promise((resolve) => setTimeout(resolve, 0));1140 }1141 1142 // Calculate streak data1143 const streakData = this.calculateStreaks(Object.keys(heatmap));1144 1145 // Calculate longest work session and total hours1146 let longestWorkDuration = 0;1147 let longestWorkDate: string | null = null;1148 let totalDurationMs = 0;1149 1150 const sessionIds = Object.keys(sessionStartTimes);1151 const totalSessions = sessionIds.length;1152 1153 for (const sessionId of sessionIds) {1154 const start = sessionStartTimes[sessionId];1155 const end = sessionEndTimes[sessionId];1156 const durationMs = end.getTime() - start.getTime();1157 const durationMinutes = Math.round(durationMs / (1000 * 60));1158 1159 totalDurationMs += durationMs;1160 1161 if (durationMinutes > longestWorkDuration) {1162 longestWorkDuration = durationMinutes;1163 longestWorkDate = this.formatDate(start);1164 }1165 }1166 1167 const totalHours = Math.round(totalDurationMs / (1000 * 60 * 60));1168 1169 // Calculate latest active time1170 let latestActiveTime: string | null = null;1171 let latestTimestamp = new Date(0);1172 for (const dateStr in heatmap) {1173 const date = new Date(dateStr);1174 if (date > latestTimestamp) {1175 latestTimestamp = date;1176 latestActiveTime = date.toLocaleTimeString([], {1177 hour: '2-digit',1178 minute: '2-digit',1179 });1180 }1181 }1182 1183 // Calculate top tools1184 const topTools = Object.entries(toolUsage)1185 .sort((a, b) => b[1] - a[1])1186 .slice(0, 10);1187 1188 return {1189 heatmap,1190 currentStreak: streakData.currentStreak,1191 longestStreak: streakData.longestStreak,1192 longestWorkDate,1193 longestWorkDuration,1194 activeHours,1195 latestActiveTime,1196 totalSessions,1197 totalMessages,1198 totalHours,1199 topTools,1200 totalLinesAdded,