Building a Video AI Interview Analyzer with Twelve Labs

Twelvelabs_API=your_api_key_here
API_URL=your_api_url_here
INDEX_ID=your_index_id_here

.
├── app.py
├── requirements.txt
├── static
│   ├── css
│   │   └── style.css
│   └── js
│       └── main.js
├── templates
│   └── index.html
└── uploads

# Importing the necessary module
import os
import json
import random
from flask import Flask, render_template, request, jsonify
from twelvelabs import TwelveLabs
from twelvelabs.models.task import Task
import requests
from dotenv import load_dotenv

# Load environment variables
load_dotenv()

# Initialize the Flask app
app = Flask(__name__)

# Get the API credentials from env variables
API_KEY = os.getenv('API_KEY')
API_URL = os.getenv('API_URL')
index_id = os.getenv('index_id')

# Initialize TwelveLabs client
client = TwelveLabs(api_key=API_KEY)

# List of the Aptitude Interview Questions
INTERVIEW_QUESTIONS = [
    "Tell me about yourself.",
    "What are your greatest strengths?",
    "What do you consider to be your weaknesses?",
    "Where do you see yourself in five years?",
    "Why should we hire you?",
    "What motivates you?",
    "What are your career goals?",
    "How do you work in a team?",
    "What's your leadership style?"
]


# Utility function to check API connection
def check_api_connection():
    try:
        response = requests.get(API_URL, headers={"x-api-key": API_KEY})
        return response.status_code == 200
    except requests.RequestException as e:
        print(f"API connection check failed. Error: {str(e)}")
        return False

# Utitliy function to process API response
def process_api_response(data):
    # Initialize default processed data structure
    processed_data = {
        "confidence": "N/A",
        "clarity": "N/A",
        "speech_rate": "N/A",
        "eye_contact": "N/A",
        "body_language": "N/A",
        "voice_tone": "N/A",
        "imp_points": []
    }

    # Handling the string input (convert to dict, if possible)
    if isinstance(data, str):
        try:
            cleaned_data = data.replace("```json", "").replace("```", "").strip()
            data = json.loads(cleaned_data)
        except json.JSONDecodeError as e:
            print(f"Error decoding JSON - {e}")
            return processed_data

    # Extract data from dict
    if isinstance(data, dict):
        for key in processed_data.keys():
            processed_data[key] = data.get(key, "N/A")
    return processed_data

# Main Page Route
@app.route('/')
def index():
    return render_template('index.html')

# API route to get a random interview question from list
@app.route('/get_question')
def get_question():
    question = random.choice(INTERVIEW_QUESTIONS)
    return jsonify({"question": question})

# Route for upload
@app.route('/upload', methods=['POST'])
def upload():
    # Checking of the API connection before proceeding
    if not check_api_connection():
        return jsonify({"error": "Failed to connect to the Twelve Labs API."}), 500

    # Validate video file in request
    if 'video' not in request.files:
        return jsonify({"error": "No video file provided"}), 400
    video = request.files['video']
    if video.filename == '':
        return jsonify({"error": "No video file selected"}), 400

    # Save uploaded video file
    video_path = os.path.join('uploads', 'interview.mp4')
    video.save(video_path)
    
    # Verifying whether the video file was saved successfully
    if not os.path.exists(video_path):
        return jsonify({"error": "Failed to save video file"}), 500

    file_size = os.path.getsize(video_path)
    print(f"Uploaded video file size: {file_size} bytes")

    if file_size == 0:
        return jsonify({"error": "Uploaded video file is empty"}), 500

    try:

       # Create and wait for indexing task
        task = client.task.create(index_id=index_id, file=video_path)
        task.wait_for_done(sleep_interval=5)
        if task.status != "ready":
            return jsonify({"error": f"Indexing failed with status {task.status}"}), 500


        # Generate text analysis from the video        
        result = client.generate.text(
            video_id=task.video_id,
            prompt="""You're an Interviewer, Analyze the video clip of the interview answer for the question - {question}.
            If the face is not present in the video then do provide the lower points in all categories, Do provide less than 5 for all the other categories if the face is not visible in the video.

            Do provide the response in the json format with the number assigned as the value. 
            After analyzing from 1-10. The keys of the json as confidence, clarity, speech_rate, eye_contact, body_language, voice_tone, relevant_to_question, imp_points.
            The imp_points will contain the exact sentence in a summarized points by the speaker, also do remove the filler words and provide it in a list format which is important from video."""
        )
        
        # Process and return the API response
        print("Raw API Response:", json.dumps(result.data, indent=2))
        processed_data = process_api_response(result.data)
        print("Processed data:", json.dumps(processed_data, indent=2))
        return jsonify(processed_data), 200
    except Exception as e:

        # To Handle any errors during processing
        print(f"Error processing video: {str(e)}")
        return jsonify({"error": f"Error processing video: {str(e)}"}), 500


# Run the Flask app
if __name__ == '__main__':
    os.makedirs('uploads', exist_ok=True) # To ensure whether the uploads directory exists
    app.run(debug=True)

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>AI Interview Analyzer</title>
    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;600&display=swap" rel="stylesheet">
    <link rel="stylesheet" href="{{ url_for('static', filename='css/style.css') }}">
</head>
<body>
    <div class="container">
        <header>
            <h1>AI Interview Analyzer</h1>
            <p>Enhance Your Interview Skills with AI Powered Feedback</p>
        </header>
        <main>
            
            <section id="start-section" class="card">
                <h2>Ready to Start Your Interview?</h2>
                <p>Click the button below to begin your AI powered interview experience.</vp>
                <button id="startInterviewButton" class="btn btn-primary">Start Interview</button>
            </section>
            
            
            <section id="interview-section" class="card hidden">
            
                <div id="question-container">
                    <h2>Interview Question -</h2>
                    <p id="interview-question"></p>
                </div>

                
                <div id="timer-container">
                    <div id="preparation-timer" class="timer">
                        Preparation Time - <span id="prep-time">00:20</span>
                    </div>
                    <div id="recording-timer" class="timer hidden">
                        Recording Time - <span id="rec-time">00:30</span>
                    </div>
                </div>

               
                <div id="video-container" class="hidden">
                    <video id="videoElement" autoplay muted></video>
                </div>

                
                <div id="status-message" class="status"></div>
            </section>
            
            <section id="result-section" class="card hidden">
            
                <h2>Analysis Results</h2>
                <div id="result" class="result-container">
                    <div id="loading-message" class="loading">Analyzing your response...</div>
                    <small>It may take around 3-4 mins</small>
                </div>
            </section>
        </main>
    </div>
    <script src="{{ url_for('static', filename='js/main.js') }}"></script>
</body>
</html>

document.addEventListener('DOMContentLoaded', () => {

    // Object to store all DOM elements
    const elements = {
        startButton: document.getElementById('startInterviewButton'),
        sections: {
            start: document.getElementById('start-section'),
            interview: document.getElementById('interview-section'),
            result: document.getElementById('result-section')
        },
        video: document.getElementById('videoElement'),
        videoContainer: document.getElementById('video-container'),
        question: document.getElementById('interview-question'),
        timers: {
            prep: document.getElementById('preparation-timer'),
            rec: document.getElementById('recording-timer')
        },
        timerDisplays: {
            prep: document.getElementById('prep-time'),
            rec: document.getElementById('rec-time')
        },
        status: document.getElementById('status-message'),
        result: document.getElementById('result')
    };
    
    // Variables for media recording
    let mediaRecorder;
    let recordedChunks = [];
    const prepTime = 20; // Interview Preparation time in seconds
    const recTime = 30;  // Video Interview Recording time in seconds
    let currentTimer;

   // Function to set up camera and media recorder
    async function setupCamera() {
        try {

            // Request access to user's camera and microphone
            const stream = await navigator.mediaDevices.getUserMedia({ 
                video: { width: 640, height: 480 },
                audio: true 
            });
            elements.video.srcObject = stream;
            
            // Set up media recorder
            const options = { mimeType: 'video/mp4' };
            mediaRecorder = new MediaRecorder(stream, options);


            // Event handler for when data is available
            mediaRecorder.ondataavailable = (event) => {
                if (event.data.size > 0) {
                    recordedChunks.push(event.data);
                }
            };

            // Event handler for when recording stops
            mediaRecorder.onstop = () => {
                const blob = new Blob(recordedChunks, { type: 'video/mp4' });
                console.log('Recording stopped. Blob size:', blob.size, 'bytes');
                if (blob.size > 0) {
                    uploadVideo(blob);
                } else {
                    showError("Recording failed: No data captured.");
                }
            };
        } catch (error) {
            console.error('Error accessing camera:', error);
            showError('Unable to access camera. Please ensure you have given permission and try again.');
        }
    }


    // Utility function to show a specific section and hide others
    function showSection(section) {
        Object.values(elements.sections).forEach(s => s.classList.add('hidden'));
        elements.sections[section].classList.remove('hidden');
    }


    // Utility function to update timer display
    function updateTimer(timerElement, time) {
        const minutes = Math.floor(time / 60);
        const seconds = time % 60;
        timerElement.textContent = `${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}`;
    }


    // Function to start and manage timer
    function startTimer(phase) {
        let timeLeft = phase === 'prep' ? prepTime : recTime;
        updateTimer(elements.timerDisplays[phase], timeLeft);
        elements.timers[phase].classList.remove('hidden');
        
        return setInterval(() => {
            timeLeft--;
            updateTimer(elements.timerDisplays[phase], timeLeft);
            if (timeLeft <= 0) {
                clearInterval(currentTimer);
                elements.timers[phase].classList.add('hidden');
                if (phase === 'prep') startRecording();
                else stopRecording();
            }
        }, 1000);
    }


    // Function to start preparation timer
    function startPreparationTimer() {
        showSection('interview');
        elements.status.textContent = "Prepare your answer...";
        currentTimer = startTimer('prep');
    }


    // Utitlity function to start recording
    function startRecording() {
        elements.videoContainer.classList.remove('hidden');
        recordedChunks = [];
        mediaRecorder.start(1000); // Record in 1-second chunks
        elements.status.textContent = "Recording in progress...";
        currentTimer = startTimer('rec');
        console.log('Recording started');
    }

    // Utitlity function to stop recording
    function stopRecording() {
        mediaRecorder.stop();
        elements.status.textContent = "Processing your response...";
        showSection('result');
        console.log('Recording stopped');
    }

    // Utitlity function to upload recorded video
    function uploadVideo(blob) {
        console.log('Uploading video. Blob size:', blob.size, 'bytes');
        const formData = new FormData();
        formData.append('video', blob, 'interview.mp4');

        fetch('/upload', {
            method: 'POST',
            body: formData
        })
        .then(response => {
            if (!response.ok) {
                return response.json().then(err => {
                    throw new Error(err.error || `HTTP error! status: ${response.status}`);
                });
            }
            return response.json();
        })
        .then(data => {
            console.log('Received data:', data);
            displayResults(data);
        })
        .catch(error => {
            console.error('Error:', error);
            showError(error.message);
        });
    }


    // Utitlity function to display analysis results
    function displayResults(data) {
        let resultHTML = '<h3>Analysis Results:</h3>';
        
        if (data.error) {
            resultHTML += `<p class="error">Error: ${data.error}</p>`;
        } else {
            resultHTML += '<div class="score-grid">';
            const metrics = [
                { key: 'confidence', label: 'Confidence' },
                { key: 'clarity', label: 'Clarity' },
                { key: 'speech_rate', label: 'Speech Rate' },
                { key: 'eye_contact', label: 'Eye Contact' },
                { key: 'body_language', label: 'Body Language' },
                { key: 'voice_tone', label: 'Voice Tone' }
            ];
            
            metrics.forEach(metric => {
                resultHTML += `
                    <div class="score">
                        <span class="score-label">${metric.label}</span>
                        <span class="score-value">${data[metric.key]}/10</span>
                    </div>
                `;
            });
            resultHTML += '</div>';
            
            if (data.imp_points && data.imp_points.length > 0) {
                resultHTML += '<h4>Key Points:</h4><ul>';
                data.imp_points.forEach(point => {
                    resultHTML += `<li>${point}</li>`;
                });
                resultHTML += '</ul>';
            } else {
                resultHTML += '<p>No key points found in the analysis.</p>';
            }
        }

        elements.result.innerHTML = resultHTML;
    }


    // Utitlity function to display error messages
    function showError(message) {
        elements.result.innerHTML = `
            <p class="error">Error: ${message}</p>
            <p>Please try again. If the problem persists, ensure you're recording for the full time and that your video and audio are working correctly.</p>
        `;
    }

    elements.startButton.addEventListener('click', () => {
        setupCamera().then(() => {
            fetch('/get_question')
                .then(response => response.json())
                .then(data => {
                    elements.question.textContent = data.question;
                    startPreparationTimer();
                })
                .catch(error => {
                    console.error('Error fetching question:', error);
                    showError('Failed to fetch interview question. Please try again.');
                });
        });
    });
});