import typing import gradio as gr import matplotlib.pyplot as plt import numpy as np import spaces import torch import torch.nn as nn from transformers import Wav2Vec2Processor from transformers.models.wav2vec2.modeling_wav2vec2 import Wav2Vec2Model from transformers.models.wav2vec2.modeling_wav2vec2 import Wav2Vec2PreTrainedModel import audiofile import audresample device = 0 if torch.cuda.is_available() else "cpu" duration = 2 # limit processing of audio age_gender_model_name = "audeering/wav2vec2-large-robust-24-ft-age-gender" expression_model_name = "audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim" class AgeGenderHead(nn.Module): r"""Age-gender model head.""" def __init__(self, config, num_labels): super().__init__() self.dense = nn.Linear(config.hidden_size, config.hidden_size) self.dropout = nn.Dropout(config.final_dropout) self.out_proj = nn.Linear(config.hidden_size, num_labels) def forward(self, features, **kwargs): x = features x = self.dropout(x) x = self.dense(x) x = torch.tanh(x) x = self.dropout(x) x = self.out_proj(x) return x class AgeGenderModel(Wav2Vec2PreTrainedModel): r"""Age-gender recognition model.""" def __init__(self, config): super().__init__(config) self.config = config self.wav2vec2 = Wav2Vec2Model(config) self.age = AgeGenderHead(config, 1) self.gender = AgeGenderHead(config, 3) self.init_weights() def forward( self, input_values, ): outputs = self.wav2vec2(input_values) hidden_states = outputs[0] hidden_states = torch.mean(hidden_states, dim=1) logits_age = self.age(hidden_states) logits_gender = torch.softmax(self.gender(hidden_states), dim=1) return hidden_states, logits_age, logits_gender class ExpressionHead(nn.Module): r"""Expression model head.""" def __init__(self, config): super().__init__() self.dense = nn.Linear(config.hidden_size, config.hidden_size) self.dropout = nn.Dropout(config.final_dropout) self.out_proj = nn.Linear(config.hidden_size, config.num_labels) def forward(self, features, **kwargs): x = features x = self.dropout(x) x = self.dense(x) x = torch.tanh(x) x = self.dropout(x) x = self.out_proj(x) return x class ExpressionModel(Wav2Vec2PreTrainedModel): r"""speech expression model.""" def __init__(self, config): super().__init__(config) self.config = config self.wav2vec2 = Wav2Vec2Model(config) self.classifier = ExpressionHead(config) self.init_weights() def forward(self, input_values): outputs = self.wav2vec2(input_values) hidden_states = outputs[0] hidden_states = torch.mean(hidden_states, dim=1) logits = self.classifier(hidden_states) return hidden_states, logits # Load models from hub age_gender_processor = Wav2Vec2Processor.from_pretrained(age_gender_model_name) age_gender_model = AgeGenderModel.from_pretrained(age_gender_model_name) expression_processor = Wav2Vec2Processor.from_pretrained(expression_model_name) expression_model = ExpressionModel.from_pretrained(expression_model_name) def process_func(x: np.ndarray, sampling_rate: int) -> typing.Tuple[str, dict, str]: r"""Predict age and gender or extract embeddings from raw audio signal.""" # run through processor to normalize signal # always returns a batch, so we just get the first entry # then we put it on the device results = [] for processor, model in zip( [age_gender_processor, expression_processor], [age_gender_model, expression_model], ): y = processor(x, sampling_rate=sampling_rate) y = y['input_values'][0] y = y.reshape(1, -1) y = torch.from_numpy(y).to(device) # run through model with torch.no_grad(): y = model(y) if len(y) == 3: # Age-gender model y = torch.hstack([y[1], y[2]]) else: # Expression model y = y[1] # convert to numpy y = y.detach().cpu().numpy() results.append(y[0]) # Plot A/D/V values plot_expression(results[1][0], results[1][1], results[1][2]) expression_file = "expression.png" plt.savefig(expression_file) return ( f"{round(100 * results[0][0])} years", # age { "female": results[0][1], "male": results[0][2], "child": results[0][3], }, expression_file, ) @spaces.GPU def recognize(input_file: str) -> typing.Tuple[str, dict, str]: # sampling_rate, signal = input_microphone # signal = signal.astype(np.float32, order="C") / 32768.0 if input_file is None: raise gr.Error( "No audio file submitted! " "Please upload or record an audio file " "before submitting your request." ) signal, sampling_rate = audiofile.read(input_file, duration=duration) # Resample to sampling rate supported byu the models target_rate = 16000 signal = audresample.resample(signal, sampling_rate, target_rate) return process_func(signal, target_rate) def plot_expression(arousal, dominance, valence): r"""3D pixel plot of arousal, dominance, valence.""" # Voxels per dimension voxels = 7 # Create voxel grid x, y, z = np.indices((voxels + 1, voxels + 1, voxels + 1)) voxel = ( (x == round(arousal * voxels)) & (y == round(dominance * voxels)) & (z == round(valence * voxels)) ) projection = ( (x == round(arousal * voxels)) & (y == round(dominance * voxels)) & (z < round(valence * voxels)) ) colors = np.empty((voxel | projection).shape, dtype=object) colors[voxel] = "#fcb06c" colors[projection] = "#fed7a9" ax = plt.figure().add_subplot(projection='3d') ax.voxels(voxel | projection, facecolors=colors, edgecolor='k') ax.set_xlim([0, voxels]) ax.set_ylim([0, voxels]) ax.set_zlim([0, voxels]) ax.set_aspect("equal") ax.set_xlabel("arousal", fontsize="large", labelpad=0) ax.set_ylabel("dominance", fontsize="large", labelpad=0) ax.set_zlabel("valence", fontsize="large", labelpad=0) ax.set_xticks( list(range(voxels + 1)), labels=[0, None, None, None, None, None, None, 1], verticalalignment="bottom", ) ax.set_yticks( list(range(voxels + 1)), labels=[0, None, None, None, None, None, None, 1], verticalalignment="bottom", ) ax.set_zticks( list(range(voxels + 1)), labels=[0, None, None, None, None, None, None, 1], verticalalignment="top", ) description = ( "Estimate **age**, **gender**, and **expression** " "of the speaker contained in an audio file or microphone recording. \n" f"The model [{age_gender_model_name}]" f"(https://huggingface.co/{age_gender_model_name}) " "recognises age and gender, " f"whereas [{expression_model_name}]" f"(https://huggingface.co/{expression_model_name}) " "recognises the expression dimensions arousal, dominance, and valence. " ) with gr.Blocks() as demo: with gr.Tab(label="Speech analysis"): with gr.Row(): with gr.Column(): gr.Markdown(description) input = gr.Audio( sources=["upload", "microphone"], type="filepath", label="Audio input", min_length=0.025, # seconds ) gr.Examples( [ "female-46-neutral.wav", "female-20-happy.wav", "male-60-angry.wav", "male-27-sad.wav", ], [input], label="Examples from CREMA-D, ODbL v1.0 license", ) gr.Markdown("Only the first two seconds of the audio will be processed.") submit_btn = gr.Button(value="Submit") with gr.Column(): output_age = gr.Textbox(label="Age") output_gender = gr.Label(label="Gender") output_expression = gr.Image(label="Expression") outputs = [output_age, output_gender, output_expression] submit_btn.click(recognize, input, outputs) demo.launch(debug=True)