590 lines
17 KiB
TypeScript
590 lines
17 KiB
TypeScript
import { GluegunMenuToolbox } from '@lenne.tech/gluegun-menu';
|
|
import chalk = require('chalk');
|
|
import { getSettings } from '../../../globals';
|
|
const XLSX = require('xlsx');
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const { Select } = require('enquirer');
|
|
const { GoogleGenerativeAI } = require('@google/generative-ai');
|
|
const OpenAI = require('openai');
|
|
|
|
// Debug mode: set to true to save all AI model responses to JSON files
|
|
const DEBUG_MODE = false;
|
|
|
|
// Function to save debug response to JSON file
|
|
function saveDebugResponse(
|
|
response: string,
|
|
debugDir: string,
|
|
filename: string,
|
|
metadata: any = {},
|
|
): void {
|
|
if (!DEBUG_MODE) return;
|
|
|
|
try {
|
|
// Ensure debug directory exists
|
|
if (!fs.existsSync(debugDir)) {
|
|
fs.mkdirSync(debugDir, { recursive: true });
|
|
}
|
|
|
|
const debugData = {
|
|
timestamp: new Date().toISOString(),
|
|
response: response,
|
|
...metadata,
|
|
};
|
|
|
|
const filePath = path.join(debugDir, filename);
|
|
fs.writeFileSync(filePath, JSON.stringify(debugData, null, 2), 'utf8');
|
|
} catch (error) {
|
|
console.error('Error saving debug response:', error.message);
|
|
}
|
|
}
|
|
|
|
// Function to extract JSON from text that might be wrapped in markdown code blocks
|
|
function extractJSON(text: string): string {
|
|
const jsonRegex = /```(?:json)?\s*([\s\S]*?)```/;
|
|
const match = text.match(jsonRegex);
|
|
|
|
if (match && match[1]) {
|
|
return match[1].trim();
|
|
}
|
|
|
|
return text.trim();
|
|
}
|
|
|
|
// Function to validate and fix JSON if needed
|
|
async function validateAndFixJSON(
|
|
response: string,
|
|
processorFunction: (
|
|
input: string,
|
|
prompt: string,
|
|
apiKey: string,
|
|
model: string,
|
|
debugDir?: string,
|
|
debugMetadata?: any,
|
|
) => Promise<string | null>,
|
|
apiKey: string,
|
|
model: string,
|
|
debugDir?: string,
|
|
debugMetadata?: any,
|
|
): Promise<{ isValid: boolean; json: any; rawText: string } | null> {
|
|
// Try to extract and parse JSON from the response
|
|
const extractedJSON = extractJSON(response);
|
|
|
|
try {
|
|
const parsed = JSON.parse(extractedJSON);
|
|
return { isValid: true, json: parsed, rawText: extractedJSON };
|
|
} catch (error) {
|
|
// JSON is invalid, try to fix it once
|
|
console.log(
|
|
chalk.yellow('⚠️ Invalid JSON detected, attempting to fix...'),
|
|
);
|
|
|
|
const fixPrompt = `The following text should be valid JSON but contains errors. Please fix it to be valid JSON format.
|
|
|
|
IMPORTANT INSTRUCTIONS:
|
|
- Output ONLY valid JSON, nothing else
|
|
- Do not add any explanations, comments, or markdown formatting
|
|
- Do not wrap the output in code blocks or backticks
|
|
- Preserve all the data from the original text
|
|
- Fix any syntax errors like missing commas, quotes, brackets, or braces
|
|
- Ensure all string values are properly quoted
|
|
- Ensure all keys are properly quoted
|
|
- Remove any trailing commas
|
|
- Make sure the JSON structure is complete and properly closed
|
|
|
|
BROKEN TEXT TO FIX:
|
|
${response}
|
|
|
|
OUTPUT ONLY THE FIXED JSON:`;
|
|
|
|
const fixedResponse = await processorFunction(
|
|
'',
|
|
fixPrompt,
|
|
apiKey,
|
|
model,
|
|
debugDir,
|
|
{ ...debugMetadata, isFixAttempt: true, originalResponse: response },
|
|
);
|
|
|
|
if (!fixedResponse) {
|
|
return null;
|
|
}
|
|
|
|
// Try to parse the fixed response
|
|
const fixedExtractedJSON = extractJSON(fixedResponse);
|
|
try {
|
|
const parsed = JSON.parse(fixedExtractedJSON);
|
|
console.log(chalk.green('✓ Successfully fixed invalid JSON'));
|
|
return { isValid: true, json: parsed, rawText: fixedExtractedJSON };
|
|
} catch (fixError) {
|
|
console.error(chalk.red('✗ Failed to fix JSON even after retry'));
|
|
return null;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Process text with Gemini AI
|
|
async function processWithGemini(
|
|
inputText: string,
|
|
promptTemplate: string,
|
|
apiKey: string,
|
|
model: string,
|
|
debugDir?: string,
|
|
debugMetadata?: any,
|
|
) {
|
|
const generationConfig = {
|
|
temperature: 0.7, // Slightly lower for more consistent JSON output
|
|
topP: 0.95,
|
|
topK: 40,
|
|
maxOutputTokens: 32768, // Significantly increased for large translation responses (Gemini 2.5 Pro supports up to 65,536)
|
|
};
|
|
|
|
const genAI = new GoogleGenerativeAI(apiKey);
|
|
const geminiModel = genAI.getGenerativeModel({
|
|
model: model,
|
|
generationConfig,
|
|
});
|
|
|
|
try {
|
|
// If inputText is provided, replace {input} placeholder, otherwise use prompt as-is
|
|
const prompt = inputText
|
|
? promptTemplate.replace('{input}', inputText)
|
|
: promptTemplate;
|
|
const result = await geminiModel.generateContent(prompt);
|
|
const response = result.response;
|
|
const responseText = response.text();
|
|
|
|
// Save debug response if debug mode is enabled
|
|
if (DEBUG_MODE && debugDir) {
|
|
const filename = `gemini_${Date.now()}_${debugMetadata?.rowNumber || 'unknown'}.json`;
|
|
saveDebugResponse(responseText, debugDir, filename, {
|
|
provider: 'Gemini',
|
|
model: model,
|
|
inputText: inputText,
|
|
prompt: prompt,
|
|
...debugMetadata,
|
|
});
|
|
}
|
|
|
|
return responseText;
|
|
} catch (error) {
|
|
console.error('Error processing with Gemini:', error.message);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// Process text with OpenAI
|
|
async function processWithOpenAI(
|
|
inputText: string,
|
|
promptTemplate: string,
|
|
apiKey: string,
|
|
model: string,
|
|
debugDir?: string,
|
|
debugMetadata?: any,
|
|
) {
|
|
const openai = new OpenAI({ apiKey });
|
|
|
|
try {
|
|
// If inputText is provided, replace {input} placeholder, otherwise use prompt as-is
|
|
const prompt = inputText
|
|
? promptTemplate.replace('{input}', inputText)
|
|
: promptTemplate;
|
|
const response = await openai.chat.completions.create({
|
|
model: model,
|
|
messages: [{ role: 'user', content: prompt }],
|
|
});
|
|
|
|
const responseText = response.choices[0]?.message?.content || null;
|
|
|
|
// Save debug response if debug mode is enabled
|
|
if (DEBUG_MODE && debugDir && responseText) {
|
|
const filename = `openai_${Date.now()}_${debugMetadata?.rowNumber || 'unknown'}.json`;
|
|
saveDebugResponse(responseText, debugDir, filename, {
|
|
provider: 'OpenAI',
|
|
model: model,
|
|
inputText: inputText,
|
|
prompt: prompt,
|
|
...debugMetadata,
|
|
});
|
|
}
|
|
|
|
return responseText;
|
|
} catch (error) {
|
|
console.error('Error processing with OpenAI:', error.message);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
module.exports = {
|
|
name: 'translate_excel',
|
|
alias: ['te'],
|
|
description: 'Translate Excel files using AI (te)',
|
|
hidden: false,
|
|
run: async (toolbox: GluegunMenuToolbox) => {
|
|
const { print, filesystem } = toolbox;
|
|
|
|
print.info(chalk.cyan('📊 Excel Translation Tool'));
|
|
print.info('');
|
|
|
|
// Get settings to check for API keys
|
|
const config = await getSettings(toolbox);
|
|
if (!config) {
|
|
print.error(
|
|
chalk.red('❌ No configuration found. Please run pwcli setup settings'),
|
|
);
|
|
return;
|
|
}
|
|
|
|
// Check which API keys are configured
|
|
const hasGemini = config.llm_api_keys?.gemini_api_key;
|
|
const hasOpenAI = config.llm_api_keys?.openai_api_key;
|
|
|
|
if (!hasGemini && !hasOpenAI) {
|
|
print.error(
|
|
chalk.red(
|
|
'❌ No AI API keys configured. Please run pwcli setup settings',
|
|
),
|
|
);
|
|
return;
|
|
}
|
|
|
|
// Step 1: Find Excel files in current directory
|
|
const currentDir = process.cwd();
|
|
const files = fs.readdirSync(currentDir);
|
|
const excelFiles = files.filter((file: string) =>
|
|
file.toLowerCase().endsWith('.xlsx'),
|
|
);
|
|
|
|
if (excelFiles.length === 0) {
|
|
print.error(
|
|
chalk.red('❌ No Excel (.xlsx) files found in current directory'),
|
|
);
|
|
return;
|
|
}
|
|
|
|
// Step 2: Let user select an Excel file
|
|
const filePrompt = new Select({
|
|
name: 'excelFile',
|
|
message: 'Select the Excel file to translate:',
|
|
choices: excelFiles,
|
|
});
|
|
|
|
const selectedFile = await filePrompt.run();
|
|
print.info(chalk.gray(`Selected file: ${selectedFile}`));
|
|
print.info('');
|
|
|
|
// Step 3: Find prompt .txt files in translate_excel folder
|
|
const promptDir = path.join(__dirname, '.');
|
|
const promptFiles = fs
|
|
.readdirSync(promptDir)
|
|
.filter((file: string) => file.toLowerCase().endsWith('.txt'));
|
|
|
|
if (promptFiles.length === 0) {
|
|
print.error(
|
|
chalk.red('❌ No prompt (.txt) files found in translate_excel folder'),
|
|
);
|
|
return;
|
|
}
|
|
|
|
// Create a mapping of display names to actual filenames
|
|
const promptDisplayNames = promptFiles.map((file: string) => {
|
|
// Remove .txt extension and convert to Title Case
|
|
const nameWithoutExt = file.replace(/\.txt$/i, '');
|
|
// Replace underscores and hyphens with spaces, then title case
|
|
return nameWithoutExt
|
|
.replace(/[_-]/g, ' ')
|
|
.split(' ')
|
|
.map(
|
|
(word) => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase(),
|
|
)
|
|
.join(' ');
|
|
});
|
|
|
|
// Create mapping object for later use
|
|
const promptMapping = {};
|
|
promptFiles.forEach((file: string, index: number) => {
|
|
promptMapping[promptDisplayNames[index]] = file;
|
|
});
|
|
|
|
// Step 4: Let user select a prompt file
|
|
const promptPrompt = new Select({
|
|
name: 'promptFile',
|
|
message: 'Select the translation prompt to use:',
|
|
choices: promptDisplayNames,
|
|
});
|
|
|
|
const selectedPromptDisplay = await promptPrompt.run();
|
|
const selectedPromptFile = promptMapping[selectedPromptDisplay];
|
|
const promptPath = path.join(promptDir, selectedPromptFile);
|
|
const promptTemplate = fs.readFileSync(promptPath, 'utf8');
|
|
print.info(chalk.gray(`Selected prompt: ${selectedPromptDisplay}`));
|
|
print.info('');
|
|
|
|
// Step 5: Let user select AI provider
|
|
const providerChoices = [];
|
|
if (hasGemini) providerChoices.push('Gemini');
|
|
if (hasOpenAI) providerChoices.push('OpenAI');
|
|
|
|
const providerPrompt = new Select({
|
|
name: 'provider',
|
|
message: 'Select AI provider:',
|
|
choices: providerChoices,
|
|
});
|
|
|
|
const selectedProvider = await providerPrompt.run();
|
|
print.info(chalk.gray(`Selected provider: ${selectedProvider}`));
|
|
print.info('');
|
|
|
|
// Step 6: Let user select model based on provider
|
|
let modelChoices: string[] = [];
|
|
let selectedModel = '';
|
|
|
|
if (selectedProvider === 'Gemini') {
|
|
modelChoices = [
|
|
'gemini-flash-latest',
|
|
'gemini-flash-lite-latest',
|
|
'gemini-2.5-pro',
|
|
];
|
|
const modelPrompt = new Select({
|
|
name: 'model',
|
|
message: 'Select Gemini model:',
|
|
choices: modelChoices,
|
|
});
|
|
selectedModel = await modelPrompt.run();
|
|
} else if (selectedProvider === 'OpenAI') {
|
|
modelChoices = ['gpt-5-nano', 'gpt-5-mini', 'gpt-5.1'];
|
|
const modelPrompt = new Select({
|
|
name: 'model',
|
|
message: 'Select OpenAI model:',
|
|
choices: modelChoices,
|
|
});
|
|
selectedModel = await modelPrompt.run();
|
|
}
|
|
|
|
print.info(chalk.gray(`Selected model: ${selectedModel}`));
|
|
print.info('');
|
|
|
|
// Step 7: Read and process the Excel file
|
|
const filePath = path.join(currentDir, selectedFile);
|
|
const workbook = XLSX.readFile(filePath);
|
|
|
|
// Use first sheet
|
|
const sheetName = workbook.SheetNames[0];
|
|
const worksheet = workbook.Sheets[sheetName];
|
|
|
|
// Convert to JSON
|
|
const data = XLSX.utils.sheet_to_json(worksheet);
|
|
|
|
if (data.length === 0) {
|
|
print.error(chalk.red('❌ Excel file is empty'));
|
|
return;
|
|
}
|
|
|
|
// Get original column order
|
|
const originalColumns = Object.keys(data[0]);
|
|
|
|
// Check if "English (Europe)" column exists
|
|
const inputColumn = 'English (Europe)';
|
|
if (!data[0].hasOwnProperty(inputColumn)) {
|
|
print.error(
|
|
chalk.red(
|
|
`❌ Column "${inputColumn}" not found. Available columns: ${originalColumns.join(', ')}`,
|
|
),
|
|
);
|
|
return;
|
|
}
|
|
|
|
print.info(
|
|
chalk.cyan(`Processing ${data.length} rows from sheet "${sheetName}"...`),
|
|
);
|
|
print.info('');
|
|
|
|
// Set up debug directory if debug mode is enabled
|
|
const debugDir = DEBUG_MODE
|
|
? path.join(currentDir, `${path.parse(selectedFile).name}_debug`)
|
|
: undefined;
|
|
|
|
if (DEBUG_MODE && debugDir) {
|
|
print.info(
|
|
chalk.gray(`Debug mode enabled. Saving responses to: ${debugDir}`),
|
|
);
|
|
print.info('');
|
|
}
|
|
|
|
// Step 8: Process each row with AI
|
|
for (let i = 0; i < data.length; i++) {
|
|
const row = data[i];
|
|
const inputText = row[inputColumn];
|
|
|
|
if (!inputText || inputText.trim() === '') {
|
|
print.info(
|
|
chalk.gray(`Skipping row ${i + 1}/${data.length} (empty input text)`),
|
|
);
|
|
continue;
|
|
}
|
|
|
|
print.info(
|
|
chalk.yellow(
|
|
`Processing row ${i + 1}/${data.length}: "${inputText.substring(0, 50)}${inputText.length > 50 ? '...' : ''}"`,
|
|
),
|
|
);
|
|
|
|
let aiResult = null;
|
|
let processorFunction = null;
|
|
let apiKey = null;
|
|
|
|
const debugMetadata = {
|
|
rowNumber: i + 1,
|
|
totalRows: data.length,
|
|
inputColumn: inputColumn,
|
|
selectedFile: selectedFile,
|
|
selectedPrompt: selectedPromptDisplay,
|
|
};
|
|
|
|
if (selectedProvider === 'Gemini') {
|
|
processorFunction = processWithGemini;
|
|
apiKey = config.llm_api_keys.gemini_api_key;
|
|
aiResult = await processWithGemini(
|
|
inputText,
|
|
promptTemplate,
|
|
apiKey,
|
|
selectedModel,
|
|
debugDir,
|
|
debugMetadata,
|
|
);
|
|
} else if (selectedProvider === 'OpenAI') {
|
|
processorFunction = processWithOpenAI;
|
|
apiKey = config.llm_api_keys.openai_api_key;
|
|
aiResult = await processWithOpenAI(
|
|
inputText,
|
|
promptTemplate,
|
|
apiKey,
|
|
selectedModel,
|
|
debugDir,
|
|
debugMetadata,
|
|
);
|
|
}
|
|
|
|
if (aiResult) {
|
|
// Validate and fix JSON if needed
|
|
const validationResult = await validateAndFixJSON(
|
|
aiResult,
|
|
processorFunction,
|
|
apiKey,
|
|
selectedModel,
|
|
debugDir,
|
|
debugMetadata,
|
|
);
|
|
|
|
if (validationResult && validationResult.json) {
|
|
const translationResult = validationResult.json;
|
|
|
|
// Update the row with translations from the LLM
|
|
for (const [country, translation] of Object.entries(
|
|
translationResult,
|
|
)) {
|
|
// Add translation if it's valid
|
|
if (translation !== null && translation !== undefined) {
|
|
row[country] = translation;
|
|
}
|
|
}
|
|
|
|
print.success(chalk.green(`✓ Translated row ${i + 1}`));
|
|
} else {
|
|
print.error(
|
|
chalk.red(
|
|
`❌ Error parsing AI response for row ${i + 1}: Invalid JSON even after retry attempt`,
|
|
),
|
|
);
|
|
console.error('AI response:', aiResult);
|
|
}
|
|
} else {
|
|
print.error(chalk.red(`❌ AI processing failed for row ${i + 1}`));
|
|
}
|
|
}
|
|
|
|
// Step 9: Write the translated data to a new Excel file
|
|
const parsedPath = path.parse(filePath);
|
|
const outputFilePath = path.join(
|
|
parsedPath.dir,
|
|
`${parsedPath.name}_translated${parsedPath.ext}`,
|
|
);
|
|
|
|
// Define the expected column order for output
|
|
const expectedColumnOrder = [
|
|
'name',
|
|
'category',
|
|
'Danish (Denmark)',
|
|
'German (Austria)',
|
|
'German (Switzerland)',
|
|
'German (Germany)',
|
|
'German (Luxembourg)',
|
|
'English (Australia)',
|
|
'English (Canada)',
|
|
'English (Estonia)',
|
|
'English (Europe)',
|
|
'English (United Kingdom)',
|
|
'English (Ireland)',
|
|
'English (New Zealand)',
|
|
'English (Singapore)',
|
|
'English (United States)',
|
|
'Spanish (Spain)',
|
|
'Finnish (Finland)',
|
|
'French (Belgium)',
|
|
'French (Canada)',
|
|
'French (Switzerland)',
|
|
'French (France)',
|
|
'French (Luxembourg)',
|
|
'Italian (Switzerland)',
|
|
'Italian (Italy)',
|
|
'Japanese',
|
|
'Dutch (Belgium)',
|
|
'Dutch (Netherlands)',
|
|
'Norwegian Nynorsk (Norway)',
|
|
'Polish (Poland)',
|
|
'Portuguese (Portugal)',
|
|
'Romanian (Romania)',
|
|
'Swedish (Sweden)',
|
|
];
|
|
|
|
// Build final column order: all expected columns + any extra columns from input
|
|
const finalColumnOrder = [
|
|
...expectedColumnOrder,
|
|
...originalColumns.filter((col) => !expectedColumnOrder.includes(col)),
|
|
];
|
|
|
|
// Ensure each row has all columns in the correct order (even if empty)
|
|
const normalizedData = data.map((row) => {
|
|
const normalizedRow = {};
|
|
for (const col of finalColumnOrder) {
|
|
normalizedRow[col] = row.hasOwnProperty(col) ? row[col] : '';
|
|
}
|
|
return normalizedRow;
|
|
});
|
|
|
|
// Write with the specified column order
|
|
const newWorksheet = XLSX.utils.json_to_sheet(normalizedData, {
|
|
header: finalColumnOrder,
|
|
});
|
|
|
|
const newWorkbook = XLSX.utils.book_new();
|
|
XLSX.utils.book_append_sheet(newWorkbook, newWorksheet, sheetName);
|
|
|
|
XLSX.writeFile(newWorkbook, outputFilePath);
|
|
|
|
print.info('');
|
|
print.success(
|
|
chalk.green(
|
|
`✅ Successfully exported translated data to ${outputFilePath}`,
|
|
),
|
|
);
|
|
print.info(
|
|
chalk.gray(`Processed ${data.length} rows from sheet "${sheetName}"`),
|
|
);
|
|
print.info('');
|
|
|
|
// Return to AI menu
|
|
await toolbox.menu.showMenu('ai');
|
|
},
|
|
};
|