import { GluegunMenuToolbox } from '@lenne.tech/gluegun-menu'; import chalk = require('chalk'); import { getSettings } from '../../../globals'; const XLSX = require('xlsx'); const fs = require('fs'); const path = require('path'); const { Select } = require('enquirer'); const { GoogleGenerativeAI } = require('@google/generative-ai'); const OpenAI = require('openai'); // Debug mode: set to true to save all AI model responses to JSON files const DEBUG_MODE = false; // Function to save debug response to JSON file function saveDebugResponse( response: string, debugDir: string, filename: string, metadata: any = {}, ): void { if (!DEBUG_MODE) return; try { // Ensure debug directory exists if (!fs.existsSync(debugDir)) { fs.mkdirSync(debugDir, { recursive: true }); } const debugData = { timestamp: new Date().toISOString(), response: response, ...metadata, }; const filePath = path.join(debugDir, filename); fs.writeFileSync(filePath, JSON.stringify(debugData, null, 2), 'utf8'); } catch (error) { console.error('Error saving debug response:', error.message); } } // Function to extract JSON from text that might be wrapped in markdown code blocks function extractJSON(text: string): string { const jsonRegex = /```(?:json)?\s*([\s\S]*?)```/; const match = text.match(jsonRegex); if (match && match[1]) { return match[1].trim(); } return text.trim(); } // Function to validate and fix JSON if needed async function validateAndFixJSON( response: string, processorFunction: ( input: string, prompt: string, apiKey: string, model: string, debugDir?: string, debugMetadata?: any, ) => Promise, apiKey: string, model: string, debugDir?: string, debugMetadata?: any, ): Promise<{ isValid: boolean; json: any; rawText: string } | null> { // Try to extract and parse JSON from the response const extractedJSON = extractJSON(response); try { const parsed = JSON.parse(extractedJSON); return { isValid: true, json: parsed, rawText: extractedJSON }; } catch (error) { // JSON is invalid, try to fix it once console.log( chalk.yellow('⚠️ Invalid JSON detected, attempting to fix...'), ); const fixPrompt = `The following text should be valid JSON but contains errors. Please fix it to be valid JSON format. IMPORTANT INSTRUCTIONS: - Output ONLY valid JSON, nothing else - Do not add any explanations, comments, or markdown formatting - Do not wrap the output in code blocks or backticks - Preserve all the data from the original text - Fix any syntax errors like missing commas, quotes, brackets, or braces - Ensure all string values are properly quoted - Ensure all keys are properly quoted - Remove any trailing commas - Make sure the JSON structure is complete and properly closed BROKEN TEXT TO FIX: ${response} OUTPUT ONLY THE FIXED JSON:`; const fixedResponse = await processorFunction( '', fixPrompt, apiKey, model, debugDir, { ...debugMetadata, isFixAttempt: true, originalResponse: response }, ); if (!fixedResponse) { return null; } // Try to parse the fixed response const fixedExtractedJSON = extractJSON(fixedResponse); try { const parsed = JSON.parse(fixedExtractedJSON); console.log(chalk.green('✓ Successfully fixed invalid JSON')); return { isValid: true, json: parsed, rawText: fixedExtractedJSON }; } catch (fixError) { console.error(chalk.red('✗ Failed to fix JSON even after retry')); return null; } } } // Process text with Gemini AI async function processWithGemini( inputText: string, promptTemplate: string, apiKey: string, model: string, debugDir?: string, debugMetadata?: any, ) { const generationConfig = { temperature: 0.7, // Slightly lower for more consistent JSON output topP: 0.95, topK: 40, maxOutputTokens: 32768, // Significantly increased for large translation responses (Gemini 2.5 Pro supports up to 65,536) }; const genAI = new GoogleGenerativeAI(apiKey); const geminiModel = genAI.getGenerativeModel({ model: model, generationConfig, }); try { // If inputText is provided, replace {input} placeholder, otherwise use prompt as-is const prompt = inputText ? promptTemplate.replace('{input}', inputText) : promptTemplate; const result = await geminiModel.generateContent(prompt); const response = result.response; const responseText = response.text(); // Save debug response if debug mode is enabled if (DEBUG_MODE && debugDir) { const filename = `gemini_${Date.now()}_${debugMetadata?.rowNumber || 'unknown'}.json`; saveDebugResponse(responseText, debugDir, filename, { provider: 'Gemini', model: model, inputText: inputText, prompt: prompt, ...debugMetadata, }); } return responseText; } catch (error) { console.error('Error processing with Gemini:', error.message); return null; } } // Process text with OpenAI async function processWithOpenAI( inputText: string, promptTemplate: string, apiKey: string, model: string, debugDir?: string, debugMetadata?: any, ) { const openai = new OpenAI({ apiKey }); try { // If inputText is provided, replace {input} placeholder, otherwise use prompt as-is const prompt = inputText ? promptTemplate.replace('{input}', inputText) : promptTemplate; const response = await openai.chat.completions.create({ model: model, messages: [{ role: 'user', content: prompt }], }); const responseText = response.choices[0]?.message?.content || null; // Save debug response if debug mode is enabled if (DEBUG_MODE && debugDir && responseText) { const filename = `openai_${Date.now()}_${debugMetadata?.rowNumber || 'unknown'}.json`; saveDebugResponse(responseText, debugDir, filename, { provider: 'OpenAI', model: model, inputText: inputText, prompt: prompt, ...debugMetadata, }); } return responseText; } catch (error) { console.error('Error processing with OpenAI:', error.message); return null; } } module.exports = { name: 'translate_excel', alias: ['te'], description: 'Translate Excel files using AI (te)', hidden: false, run: async (toolbox: GluegunMenuToolbox) => { const { print, filesystem } = toolbox; print.info(chalk.cyan('📊 Excel Translation Tool')); print.info(''); // Get settings to check for API keys const config = await getSettings(toolbox); if (!config) { print.error( chalk.red('❌ No configuration found. Please run pwcli setup settings'), ); return; } // Check which API keys are configured const hasGemini = config.llm_api_keys?.gemini_api_key; const hasOpenAI = config.llm_api_keys?.openai_api_key; if (!hasGemini && !hasOpenAI) { print.error( chalk.red( '❌ No AI API keys configured. Please run pwcli setup settings', ), ); return; } // Step 1: Find Excel files in current directory const currentDir = process.cwd(); const files = fs.readdirSync(currentDir); const excelFiles = files.filter((file: string) => file.toLowerCase().endsWith('.xlsx'), ); if (excelFiles.length === 0) { print.error( chalk.red('❌ No Excel (.xlsx) files found in current directory'), ); return; } // Step 2: Let user select an Excel file const filePrompt = new Select({ name: 'excelFile', message: 'Select the Excel file to translate:', choices: excelFiles, }); const selectedFile = await filePrompt.run(); print.info(chalk.gray(`Selected file: ${selectedFile}`)); print.info(''); // Step 3: Find prompt .txt files in translate_excel folder const promptDir = path.join(__dirname, '.'); const promptFiles = fs .readdirSync(promptDir) .filter((file: string) => file.toLowerCase().endsWith('.txt')); if (promptFiles.length === 0) { print.error( chalk.red('❌ No prompt (.txt) files found in translate_excel folder'), ); return; } // Create a mapping of display names to actual filenames const promptDisplayNames = promptFiles.map((file: string) => { // Remove .txt extension and convert to Title Case const nameWithoutExt = file.replace(/\.txt$/i, ''); // Replace underscores and hyphens with spaces, then title case return nameWithoutExt .replace(/[_-]/g, ' ') .split(' ') .map( (word) => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase(), ) .join(' '); }); // Create mapping object for later use const promptMapping = {}; promptFiles.forEach((file: string, index: number) => { promptMapping[promptDisplayNames[index]] = file; }); // Step 4: Let user select a prompt file const promptPrompt = new Select({ name: 'promptFile', message: 'Select the translation prompt to use:', choices: promptDisplayNames, }); const selectedPromptDisplay = await promptPrompt.run(); const selectedPromptFile = promptMapping[selectedPromptDisplay]; const promptPath = path.join(promptDir, selectedPromptFile); const promptTemplate = fs.readFileSync(promptPath, 'utf8'); print.info(chalk.gray(`Selected prompt: ${selectedPromptDisplay}`)); print.info(''); // Step 5: Let user select AI provider const providerChoices = []; if (hasGemini) providerChoices.push('Gemini'); if (hasOpenAI) providerChoices.push('OpenAI'); const providerPrompt = new Select({ name: 'provider', message: 'Select AI provider:', choices: providerChoices, }); const selectedProvider = await providerPrompt.run(); print.info(chalk.gray(`Selected provider: ${selectedProvider}`)); print.info(''); // Step 6: Let user select model based on provider let modelChoices: string[] = []; let selectedModel = ''; if (selectedProvider === 'Gemini') { modelChoices = [ 'gemini-flash-latest', 'gemini-flash-lite-latest', 'gemini-2.5-pro', ]; const modelPrompt = new Select({ name: 'model', message: 'Select Gemini model:', choices: modelChoices, }); selectedModel = await modelPrompt.run(); } else if (selectedProvider === 'OpenAI') { modelChoices = ['gpt-5-nano', 'gpt-5-mini', 'gpt-5.1']; const modelPrompt = new Select({ name: 'model', message: 'Select OpenAI model:', choices: modelChoices, }); selectedModel = await modelPrompt.run(); } print.info(chalk.gray(`Selected model: ${selectedModel}`)); print.info(''); // Step 7: Read and process the Excel file const filePath = path.join(currentDir, selectedFile); const workbook = XLSX.readFile(filePath); // Use first sheet const sheetName = workbook.SheetNames[0]; const worksheet = workbook.Sheets[sheetName]; // Convert to JSON const data = XLSX.utils.sheet_to_json(worksheet); if (data.length === 0) { print.error(chalk.red('❌ Excel file is empty')); return; } // Get original column order const originalColumns = Object.keys(data[0]); // Check if "English (Europe)" column exists const inputColumn = 'English (Europe)'; if (!data[0].hasOwnProperty(inputColumn)) { print.error( chalk.red( `❌ Column "${inputColumn}" not found. Available columns: ${originalColumns.join(', ')}`, ), ); return; } print.info( chalk.cyan(`Processing ${data.length} rows from sheet "${sheetName}"...`), ); print.info(''); // Set up debug directory if debug mode is enabled const debugDir = DEBUG_MODE ? path.join(currentDir, `${path.parse(selectedFile).name}_debug`) : undefined; if (DEBUG_MODE && debugDir) { print.info( chalk.gray(`Debug mode enabled. Saving responses to: ${debugDir}`), ); print.info(''); } // Step 8: Process each row with AI for (let i = 0; i < data.length; i++) { const row = data[i]; const inputText = row[inputColumn]; if (!inputText || inputText.trim() === '') { print.info( chalk.gray(`Skipping row ${i + 1}/${data.length} (empty input text)`), ); continue; } print.info( chalk.yellow( `Processing row ${i + 1}/${data.length}: "${inputText.substring(0, 50)}${inputText.length > 50 ? '...' : ''}"`, ), ); let aiResult = null; let processorFunction = null; let apiKey = null; const debugMetadata = { rowNumber: i + 1, totalRows: data.length, inputColumn: inputColumn, selectedFile: selectedFile, selectedPrompt: selectedPromptDisplay, }; if (selectedProvider === 'Gemini') { processorFunction = processWithGemini; apiKey = config.llm_api_keys.gemini_api_key; aiResult = await processWithGemini( inputText, promptTemplate, apiKey, selectedModel, debugDir, debugMetadata, ); } else if (selectedProvider === 'OpenAI') { processorFunction = processWithOpenAI; apiKey = config.llm_api_keys.openai_api_key; aiResult = await processWithOpenAI( inputText, promptTemplate, apiKey, selectedModel, debugDir, debugMetadata, ); } if (aiResult) { // Validate and fix JSON if needed const validationResult = await validateAndFixJSON( aiResult, processorFunction, apiKey, selectedModel, debugDir, debugMetadata, ); if (validationResult && validationResult.json) { const translationResult = validationResult.json; // Update the row with translations from the LLM for (const [country, translation] of Object.entries( translationResult, )) { // Add translation if it's valid if (translation !== null && translation !== undefined) { row[country] = translation; } } print.success(chalk.green(`✓ Translated row ${i + 1}`)); } else { print.error( chalk.red( `❌ Error parsing AI response for row ${i + 1}: Invalid JSON even after retry attempt`, ), ); console.error('AI response:', aiResult); } } else { print.error(chalk.red(`❌ AI processing failed for row ${i + 1}`)); } } // Step 9: Write the translated data to a new Excel file const parsedPath = path.parse(filePath); const outputFilePath = path.join( parsedPath.dir, `${parsedPath.name}_translated${parsedPath.ext}`, ); // Define the expected column order for output const expectedColumnOrder = [ 'name', 'category', 'Danish (Denmark)', 'German (Austria)', 'German (Switzerland)', 'German (Germany)', 'German (Luxembourg)', 'English (Australia)', 'English (Canada)', 'English (Estonia)', 'English (Europe)', 'English (United Kingdom)', 'English (Ireland)', 'English (New Zealand)', 'English (Singapore)', 'English (United States)', 'Spanish (Spain)', 'Finnish (Finland)', 'French (Belgium)', 'French (Canada)', 'French (Switzerland)', 'French (France)', 'French (Luxembourg)', 'Italian (Switzerland)', 'Italian (Italy)', 'Japanese', 'Dutch (Belgium)', 'Dutch (Netherlands)', 'Norwegian Nynorsk (Norway)', 'Polish (Poland)', 'Portuguese (Portugal)', 'Romanian (Romania)', 'Swedish (Sweden)', ]; // Build final column order: all expected columns + any extra columns from input const finalColumnOrder = [ ...expectedColumnOrder, ...originalColumns.filter((col) => !expectedColumnOrder.includes(col)), ]; // Ensure each row has all columns in the correct order (even if empty) const normalizedData = data.map((row) => { const normalizedRow = {}; for (const col of finalColumnOrder) { normalizedRow[col] = row.hasOwnProperty(col) ? row[col] : ''; } return normalizedRow; }); // Write with the specified column order const newWorksheet = XLSX.utils.json_to_sheet(normalizedData, { header: finalColumnOrder, }); const newWorkbook = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(newWorkbook, newWorksheet, sheetName); XLSX.writeFile(newWorkbook, outputFilePath); print.info(''); print.success( chalk.green( `✅ Successfully exported translated data to ${outputFilePath}`, ), ); print.info( chalk.gray(`Processed ${data.length} rows from sheet "${sheetName}"`), ); print.info(''); // Return to AI menu await toolbox.menu.showMenu('ai'); }, };