import streamlit as st from transformers import Wav2Vec2Processor, Wav2Vec2ForCTC import torch import numpy as np import soundfile as sf import io import librosa import matplotlib.pyplot as plt import librosa.display import zipfile import os from datetime import datetime # For recording from st_audiorec import st_audiorec import base64 from pydub import AudioSegment st.title("Syllables per Second Calculator") st.write( "Upload an audio file *or* record from your microphone to calculate " "the number of 'p', 't', and 'k' syllables per second." ) def get_syllables_per_second(audio_bytes): """ Processes an audio file-like object (or BytesIO) and returns syllables/sec. """ processor = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-xlsr-53-espeak-cv-ft") model = Wav2Vec2ForCTC.from_pretrained("facebook/wav2vec2-xlsr-53-espeak-cv-ft") # Read the audio data audio_input, original_sample_rate = sf.read(io.BytesIO(audio_bytes)) target_sample_rate = processor.feature_extractor.sampling_rate # Resample if needed if original_sample_rate != target_sample_rate: if audio_input.ndim > 1: audio_input = np.asarray([ librosa.resample( channel, orig_sr=original_sample_rate, target_sr=target_sample_rate ) for channel in audio_input.T ]).T else: audio_input = librosa.resample( audio_input, orig_sr=original_sample_rate, target_sr=target_sample_rate ) # Convert to mono if stereo if audio_input.ndim > 1: audio_input = np.mean(audio_input, axis=1) # Create mel spectrogram mel_spec = librosa.feature.melspectrogram( y=audio_input, sr=target_sample_rate, n_mels=128, fmax=8000 ) mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max) # Prepare input for the model input_values = processor(audio_input, return_tensors="pt").input_values # Perform inference with torch.no_grad(): logits = model(input_values).logits probabilities = torch.softmax(logits, dim=-1) predicted_ids = torch.argmax(logits, dim=-1) transcription = processor.batch_decode(predicted_ids, output_char_offsets=True) offsets = transcription["char_offsets"] print("Offsets are:", offsets) # We only care about 'p', 't', or 'k' syllable_offsets = [item for item in offsets[0] if item['char'] in ['p', 't', 'k']] if syllable_offsets: first_syllable_offset = syllable_offsets[0]['start_offset'] * 0.02 last_syllable_offset = syllable_offsets[-1]['end_offset'] * 0.02 syllable_duration = last_syllable_offset - first_syllable_offset else: syllable_duration = 0 syllable_count = len(syllable_offsets) audio_duration = len(audio_input) / target_sample_rate syllables_per_second = syllable_count / syllable_duration if syllable_duration > 0 else 0 # Plot syllables per second over time times = [] syllables_per_second_time = [] for i in range(len(syllable_offsets) - 1): start = syllable_offsets[i]['start_offset'] * 0.02 end = syllable_offsets[i + 1]['end_offset'] * 0.02 duration = end - start rate = 1 / duration if duration > 0 else 0 times.append(start) syllables_per_second_time.append(rate) plt.figure(figsize=(8, 3)) plt.plot(times, syllables_per_second_time, marker='o') plt.xlabel('Time (s)') plt.ylabel('Syllables per second') plt.title('Syllables/Second Over Time') plt.grid(True) plt.tight_layout() plt.savefig('syllables_per_second.png') plt.close() # Create a new figure for the mel spectrogram plt.figure(figsize=(12, 6)) librosa.display.specshow( mel_spec_db, sr=target_sample_rate, x_axis='time', y_axis='mel', fmax=8000 ) # Highlight p, t, k sounds for offset in syllable_offsets: start_time = offset['start_offset'] * 0.02 end_time = offset['end_offset'] * 0.02 mid_time = (start_time + end_time) / 2 plt.axvline(x=start_time, color='r', alpha=0.3, linestyle='--') plt.text(mid_time, mel_spec_db.shape[0] * 0.9, offset['char'].upper(), horizontalalignment='center', color='white', bbox=dict(facecolor='red', alpha=0.7)) plt.colorbar(format='%+2.0f dB') plt.title('Mel Spectrogram with Highlighted Syllables') plt.tight_layout() plt.savefig('mel_spectrogram.png') plt.close() # Calculate evenness and distinctness metrics syllable_stats = {} for syllable in ['p', 't', 'k']: syllable_times = [offset for offset in syllable_offsets if offset['char'] == syllable] if len(syllable_times) > 1: intervals = [(syllable_times[i+1]['start_offset'] - syllable_times[i]['start_offset']) * 0.02 for i in range(len(syllable_times)-1)] mean_interval = np.mean(intervals) std_interval = np.std(intervals) cv = (std_interval / mean_interval) if mean_interval > 0 else 0 # Debug prints for confidence calculation syllable_idx = processor.tokenizer.convert_tokens_to_ids(syllable) print(f"\nProcessing syllable: {syllable} (token_id: {syllable_idx})") confidence_scores = [] # Only look at time windows where this syllable occurs for offset in syllable_times: # Convert time offset to model timestep index time_idx = int(offset['start_offset']) prob = probabilities[0][time_idx] # Get top 5 predictions and their indices top_k_values, top_k_indices = torch.topk(prob, k=5) print(f"\nTimestep {time_idx} (time: {time_idx * 0.02:.3f}s):") print(f"Top-5 indices: {top_k_indices.tolist()}") print(f"Top-5 values: {top_k_values.tolist()}") if syllable_idx in top_k_indices: syllable_prob = prob[syllable_idx] relative_confidence = syllable_prob / top_k_values.sum() print(f"Syllable probability: {syllable_prob:.4f}") print(f"Relative confidence: {relative_confidence:.4f}") confidence_scores.append(float(relative_confidence)) else: confidence_scores.append(0.0) print("Syllable not in top-5") # Calculate mean confidence only from timesteps where syllable occurs mean_confidence = np.mean(confidence_scores) if confidence_scores else 0.0 print(f"\nFinal confidence scores for {syllable}:") print(f"Scores at syllable timestamps: {confidence_scores}") print(f"Mean confidence: {mean_confidence:.4f}") syllable_stats[syllable] = { 'count': len(syllable_times), 'mean_interval': mean_interval, 'std_interval': std_interval, 'cv': cv, 'mean_confidence': mean_confidence, 'intervals': intervals, 'confidence_scores': confidence_scores } # Create visualization for evenness and distinctness fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(12, 10)) # Color scheme colors = { 'p': '#2E86C1', # Blue 't': '#28B463', # Green 'k': '#E74C3C' # Red } # Plot 1: Evenness Analysis for syllable, stats in syllable_stats.items(): if len(stats['intervals']) > 0: # Calculate normalized intervals (deviation from mean) mean_interval = stats['mean_interval'] normalized_intervals = [(interval - mean_interval) / mean_interval * 100 for interval in stats['intervals']] # Plot normalized intervals x = range(len(normalized_intervals)) ax1.plot(x, normalized_intervals, 'o-', label=f'{syllable} (CV={stats["cv"]:.2f})', color=colors[syllable], linewidth=2, markersize=8) # Add individual point annotations for i, val in enumerate(normalized_intervals): ax1.annotate(f'{val:.1f}%', (i, val), xytext=(0, 10), textcoords='offset points', ha='center', fontsize=8) # Add reference zones for evenness ax1.axhspan(-10, 10, color='#2ECC71', alpha=0.2, label='Highly Regular (±10%)') ax1.axhspan(-30, -10, color='#F1C40F', alpha=0.2, label='Moderately Regular') ax1.axhspan(10, 30, color='#F1C40F', alpha=0.2) ax1.axhspan(-50, -30, color='#E74C3C', alpha=0.2, label='Irregular') ax1.axhspan(30, 50, color='#E74C3C', alpha=0.2) ax1.set_xlabel('Repetition Number', fontsize=12) ax1.set_ylabel('Deviation from Mean Interval (%)', fontsize=12) ax1.set_title('Timing Evenness Analysis\n(Deviation from Mean Interval)', fontsize=14, pad=20) ax1.grid(True, linestyle='--', alpha=0.7) ax1.legend(loc='upper right', bbox_to_anchor=(1.15, 1)) ax1.set_ylim(-50, 50) # Plot 2: Distinctness Analysis for syllable, stats in syllable_stats.items(): if len(stats['confidence_scores']) > 0: x = range(len(stats['confidence_scores'])) # Create gradient colors based on confidence scores colors_array = [] for score in stats['confidence_scores']: if score > 0.7: colors_array.append('#2ECC71') # Green for high confidence elif score > 0.4: colors_array.append('#F1C40F') # Yellow for medium confidence else: colors_array.append('#E74C3C') # Red for low confidence # Plot bars with gradient colors bars = ax2.bar(x, stats['confidence_scores'], label=f'{syllable} (mean={stats["mean_confidence"]:.2f})', color=colors_array, alpha=0.7) # Add value labels on top of bars for bar in bars: height = bar.get_height() ax2.text(bar.get_x() + bar.get_width()/2., height, f'{height:.2f}', ha='center', va='bottom', fontsize=8) # Add reference lines for distinctness ax2.axhline(y=0.7, color='#2ECC71', linestyle='--', alpha=0.5, label='High Distinctness') ax2.axhline(y=0.4, color='#F1C40F', linestyle='--', alpha=0.5, label='Moderate Distinctness') ax2.set_xlabel('Syllable Occurrence', fontsize=12) ax2.set_ylabel('Articulation Distinctness Score', fontsize=12) ax2.set_title('Articulation Distinctness Analysis\n(Higher Score = Clearer Articulation)', fontsize=14, pad=20) ax2.grid(True, linestyle='--', alpha=0.7) ax2.legend(loc='upper right', bbox_to_anchor=(1.15, 1)) ax2.set_ylim(0, 1) # Overall layout adjustments plt.tight_layout() plt.subplots_adjust(right=0.85) # Make room for legends plt.savefig('articulation_analysis.png', dpi=300, bbox_inches='tight') plt.close() # Update results text with new metrics results_text = f"""Syllables per Second Analysis Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} SPEED MEASUREMENTS ---------------- - Overall syllables per second: {syllables_per_second:.2f} - Total number of syllables: {syllable_count} - Total duration: {audio_duration:.2f} seconds Detailed Analysis by Syllable:""" for syllable, stats in syllable_stats.items(): results_text += f""" {syllable.upper()} Syllable Analysis: Count: {stats['count']} occurrences EVENNESS MEASUREMENTS (timing regularity) -------------------------------- - Mean interval between repetitions: {stats['mean_interval']:.3f} seconds - Variation in intervals (std dev): {stats['std_interval']:.3f} seconds - Coefficient of variation: {stats['cv']:.3f} (Lower CV = more even timing, Higher CV = more irregular timing) * CV < 0.1: Highly regular * CV 0.1-0.3: Moderately regular * CV > 0.3: Irregular DISTINCTNESS MEASUREMENTS (articulation clarity) ------------------------------------ - Mean articulation confidence: {stats['mean_confidence']:.3f} (Higher values indicate clearer articulation) * Values closer to 1.0 indicate very distinct pronunciation * Values closer to 0.0 indicate less distinct pronunciation - Confidence scores for each occurrence: {stats['confidence_scores']} RAW MEASUREMENTS -------------- - All intervals between repetitions (seconds): {stats['intervals']}""" # Print the results text to verify print("\nFinal Results Text:") print(results_text) # Create results directory if it doesn't exist os.makedirs('results', exist_ok=True) # Save the audio file audio_path = 'results/recorded_audio.wav' with open(audio_path, 'wb') as f: f.write(audio_bytes) # Save syllables per second to text file with open('results/analysis_results.txt', 'w') as f: f.write(results_text) # Create zip file zip_path = 'results/analysis_package.zip' with zipfile.ZipFile(zip_path, 'w') as zipf: zipf.write('syllables_per_second.png') zipf.write('mel_spectrogram.png') zipf.write('articulation_analysis.png') zipf.write(audio_path) zipf.write('results/analysis_results.txt') # Add download button after the visualizations with open(zip_path, 'rb') as f: st.download_button( label="Download Analysis Package", data=f, file_name="ddk_analysis_package.zip", mime="application/zip", help="Download a zip file containing the audio, visualizations, and analysis results" ) # Display all visualizations col1, col2 = st.columns(2) with col1: st.image('syllables_per_second.png') st.image('articulation_analysis.png') with col2: st.image('mel_spectrogram.png') # Display detailed metrics st.write("### Detailed Analysis") for syllable, stats in syllable_stats.items(): st.write(f"\n**{syllable.upper()} Syllable:**") st.write(f"- Count: {stats['count']}") st.write(f"- Mean interval: {stats['mean_interval']:.3f} seconds") st.write(f"- Coefficient of variation: {stats['cv']:.3f}") st.write(f"- Mean articulation confidence: {stats['mean_confidence']:.3f}") return syllables_per_second # ----------------------------- # SECTION: File Uploader # ----------------------------- uploaded_file = st.file_uploader("Choose an audio file", type=["wav", "mp3", "ogg"]) if uploaded_file is not None: with st.spinner("Processing the uploaded audio file..."): # Read entire file into bytes audio_bytes = uploaded_file.read() result = get_syllables_per_second(audio_bytes) st.write(f"**Syllables per second (uploaded):** {result:.2f}") # ----------------------------- # SECTION: Audio Recorder # ----------------------------- st.write("---") st.subheader("Or record audio from your microphone") # The st_audiorec component returns base64 encoded wav data recorded_data = st_audiorec() if recorded_data is not None: st.info("Audio recording complete. Processing ...") # Check if recorded_data is bytes or string if isinstance(recorded_data, bytes): decoded = recorded_data else: # Convert the base64 encoded data to wav audio # recorded_data is a base64 string with headers, so we split off the prefix try: header, encoded = recorded_data.split(",", 1) decoded = base64.b64decode(encoded) except AttributeError: st.error("Unexpected audio format received from recorder") st.stop() # Rest of the processing remains the same audio_segment = AudioSegment.from_file(io.BytesIO(decoded), format="wav") wav_io = io.BytesIO() audio_segment.export(wav_io, format="wav") wav_bytes = wav_io.getvalue() # Run the analysis using the same function with st.spinner("Analyzing your recorded audio..."): recorded_result = get_syllables_per_second(wav_bytes) st.write(f"**Syllables per second (recorded):** {recorded_result:.2f}")