Building TruthMeter AI: Measuring the Real Impact of AI Coding Assistants

npm install -g yo generator-code yo code  ? What type of extension? New Extension (TypeScript) ? What's the name? TruthMeter AI ? What's the identifier? truthmeter-ai ? What's the description? Measure actual AI coding assistant impact ? Initialize git repository? Yes ? Bundle with webpack? Yes ? Use ESLint? Yes

{   "compilerOptions": {     "module": "commonjs",     "target": "ES2020",     "outDir": "./out",     "lib": ["ES2020"],     "sourceMap": true,     "strict": true,     "rootDir": "./src"   } }

src/ ├── collectors/      # Data collection layer ├── analyzers/       # Analysis layer ├── calculators/     # Business logic layer ├── storage/        # Persistence layer ├── ui/             # Presentation layer └── types/          # TypeScript interfaces

private isAIGenerated(change: TextDocumentChangeEvent): boolean {   const text = change.document.getText();   const insertedLength = change.contentChanges[0]?.text.length || 0;    // Heuristic 1: Large text insertions (>100 chars at once)   if (insertedLength > 100) return true;    // Heuristic 2: Pattern matching for AI signatures   const aiPatterns = [     /\/\/ TODO: Implement/gi,     /function\s+\w+\([^)]*\)\s*{\s*\/\/ Implementation/gi,     /catch\s*\([^)]+\)\s*{\s*console\.error/gi   ];    if (aiPatterns.some(pattern => pattern.test(text))) return true;    // Heuristic 3: Rapid successive changes (AI streaming)   const timeSinceLastChange = Date.now() - this.lastChangeTime;   if (timeSinceLastChange < 100 && insertedLength > 20) return true;    return false; }

async calculateCodeChurn(): Promise<ChurnMetrics> {   const log = await this.git.log({ maxCount: 100 });   const commits = Array.from(log.all);    let totalChurn = 0;   let aiChurn = 0;    for (let i = 1; i < commits.length; i++) {     const diff = await this.git.diff([       `${commits[i].hash}..${commits[i-1].hash}`     ]);      const linesChanged = this.countLines(diff);     totalChurn += linesChanged;      // Check if changes occurred within 14 days of creation     const timeDiff = commits[i-1].date.getTime() - commits[i].date.getTime();     if (timeDiff < 14 * 24 * 60 * 60 * 1000) {       aiChurn += linesChanged;     }   }    return {     rate: aiChurn / totalChurn,     trend: this.calculateTrend(commits),     aiVsHuman: aiChurn / (totalChurn - aiChurn)   }; }

private calculateCyclomaticComplexity(text: string): number {   let complexity = 1; // Base complexity    const decisionPoints = [     /\bif\b/g, /\bfor\b/g, /\bwhile\b/g,     /\bcase\b/g, /&&/g, /\|\|/g, /\?/g   ];    for (const pattern of decisionPoints) {     const matches = text.match(pattern);     complexity += matches ? matches.length : 0;   }    return complexity; }

private detectDuplication(code: string): number {   const lines = code.split('\n');   const blockSize = 5;   const hashCounts = new Map<number, number>();    for (let i = 0; i <= lines.length - blockSize; i++) {     const block = lines.slice(i, i + blockSize).join('\n');     const hash = this.fastHash(block);     hashCounts.set(hash, (hashCounts.get(hash) || 0) + 1);   }    let duplicates = 0;   for (const count of hashCounts.values()) {     if (count > 1) duplicates += count - 1;   }    return duplicates / hashCounts.size; }

async calculate(): Promise<EconomicMetrics> {   // Based on research   const timeSaved = 2.5;      // hours/week (GitClear 2024)   const timeWasted = 3.1;     // hours/week (METR 2025)   const netTimeSaved = timeSaved - timeWasted; // -0.6 hours/week    const developerRate = 75;   // $/hour   const weeklyValue = netTimeSaved * developerRate;    // Hidden costs   const technicalDebt = 5000;   const maintenanceBurden = 2000;   const knowledgeGaps = 1500;    const monthlyLicense = 20;   const monthlyBenefit = weeklyValue * 4 - monthlyLicense;   const totalCosts = technicalDebt + maintenanceBurden + knowledgeGaps;   const roi = (monthlyBenefit * 12 - totalCosts) / (monthlyLicense * 12);    return { /* ... */ }; }

export class MockExtensionContext implements vscode.ExtensionContext {   globalState = new MockMemento();   workspaceState = new MockMemento();   subscriptions: vscode.Disposable[] = [];    extensionUri: vscode.Uri;   extensionPath: string;    // ... full implementation }

src/test/suite/ ├── storage.test.ts        # 13 tests ├── collectors.test.ts     # 21 tests ├── analyzers.test.ts      # 26 tests ├── calculators.test.ts    # 15 tests ├── ui.test.ts             # 14 tests ├── utils.test.ts          # 15 tests └── integration.test.ts    # 10 tests

test('should limit stored entries to 1000', async () => {   await storage.initialize();    // Store 1100 items   for (let i = 0; i < 1100; i++) {     await storage.store('test_key', { index: i });   }    const retrieved = await storage.get('test_key');    assert.strictEqual(retrieved.length, 1000);   assert.strictEqual(retrieved[0].index, 100); // Oldest removed });

test('should show time wasted exceeds time saved', async () => {   const result = await calculator.calculate();    // Based on research: timeWasted (3.1h) > timeSaved (2.5h)   assert.ok(result.costBenefit.timeWasted > result.costBenefit.timeSaved);   assert.ok(result.costBenefit.netValue < 0); });

{   "branches": 70,   "lines": 80,   "functions": 75,   "statements": 80 }

// BEFORE - Memory leak private events: AIEvent[] = [];  // Grows to 150MB+

// AFTER - Constant memory private events: (AIEvent | undefined)[]; private readonly MAX_EVENTS = 1000;  // ~100KB max  private addEvent(event: AIEvent) {   this.events[this.eventIndex] = event;   this.eventIndex = (this.eventIndex + 1) % this.MAX_EVENTS; }

// BEFORE - CPU spike onDidChangeTextDocument((event) => {   this.processChange(event);  // Called 100x while typing! });

// AFTER - Efficient processing private debouncers: Map<string, NodeJS.Timeout> = new Map();  onDidChangeTextDocument((event) => {   const existing = this.debouncers.get(uri);   if (existing) clearTimeout(existing);    const timer = setTimeout(() => {     this.processChange(event);  // Once after typing pause   }, 300);    this.debouncers.set(uri, timer); });

private writeQueue: Map<string, any[]> = new Map();  setInterval(() => {   for (const [key, values] of this.writeQueue) {     await this.storage.storeBatch(key, values);   } }, 5000);

const chunkSize = 5; for (let i = 0; i < files.length; i += chunkSize) {   const chunk = files.slice(i, i + chunkSize);   await Promise.all(chunk.map(f => processFile(f)));    // Yield to event loop   await new Promise(resolve => setImmediate(resolve)); }

dispose() {   this.pendingTimers.forEach(t => clearTimeout(t));   this.pendingTimers.clear();   this.debouncers.clear(); }

private trackFlowState() {   const FLOW_THRESHOLD = 15 * 60 * 1000; // 15 minutes    if (this.isActivelyTyping() && this.timeSinceLastChange < 2000) {     this.flowStateTime += this.timeSinceLastChange;   } else {     if (this.flowStateTime > FLOW_THRESHOLD) {       this.flowSessions.push(this.flowStateTime);     }     this.flowStateTime = 0;   } }

onDidChangeActiveTextEditor((editor) => {   const timeSinceLastSwitch = Date.now() - this.lastSwitchTime;    if (timeSinceLastSwitch < 5000) {  // Rapid switching     this.contextSwitches++;   }    this.lastSwitchTime = Date.now(); });

onAcceptSuggestion((suggestion) => {   const startTime = Date.now();    const disposable = onDidChangeTextDocument((event) => {     const fixTime = Date.now() - startTime;      if (fixTime > 2 * 60 * 1000) {  // >2 minutes = poor suggestion       this.poorSuggestions++;     }   }); });

const aiChurnRate = aiChurn / totalChurn; const humanChurnRate = humanChurn / totalChurn; const ratio = aiChurnRate / humanChurnRate;  // Ratio > 2 means AI code is churned 2x more than human code

vscode.window.showQuickPick([   { label: '😊 Great - AI was helpful', value: 5 },   { label: '🙂 Good - Mostly helpful', value: 4 },   { label: '😐 Neutral - Mixed results', value: 3 },   { label: '😕 Poor - More hindrance', value: 2 },   { label: '😞 Terrible - Wasted my time', value: 1 } ], {   placeHolder: 'How did this coding session feel?' });

// Detect if user rejects most suggestions in specific languages if (pythonRejectionRate > 0.8) {   vscode.window.showWarningMessage(     'You reject 80% of AI suggestions in Python files. ' +     'Consider disabling Copilot for Python to reduce distractions.'   ); }

const trends = {   productivity: this.calculateTrend(netTimeSavedHistory),   usage: this.calculateTrend(suggestionCountHistory) };  // Find the "sweet spot" where productivity peaks const sweetSpot = this.findOptimalUsageLevel(trends);

const timeframes = {   '7d': last7Days,   '30d': last30Days,   '90d': last90Days,   'all': allTime };

async trackBuildOutcome(success: boolean) {   const recentAIUsage = this.getRecentSuggestionCount();    this.buildResults.push({     success,     aiUsageLevel: recentAIUsage,     timestamp: Date.now()   });    // Analyze correlation   const correlation = this.calculateCorrelation(     buildResults.map(r => r.success),     buildResults.map(r => r.aiUsageLevel)   ); }

// Mock entire VS Code API const vscode = {   window: {     showInformationMessage: sinon.stub(),     createWebviewPanel: sinon.stub()   },   workspace: {     onDidChangeTextDocument: sinon.stub()   } };

try {   const log = await this.git.log({ maxCount: 100 }); } catch (error) {   // Fallback to basic metrics if git fails   return this.getBasicMetrics(); }

// Extension → WebView panel.webview.postMessage({   type: 'updateMetrics',   data: metrics });  // WebView → Extension panel.webview.onDidReceiveMessage((message) => {   if (message.type === 'requestData') {     this.sendMetrics(panel);   } });