工程化與框架系列（27）--前端音視頻處理

前端音視頻處理 🎥

引言

前端音視頻處理是現代Web應用中的重要組成部分，涉及音頻播放、視頻處理、流媒體傳輸等多個方面。本文將深入探討前端音視頻處理的關鍵技術和最佳實踐，幫助開發者構建高質量的多媒體應用。

音視頻技術概述

前端音視頻處理主要包括以下技術方向：

音頻處理：音頻播放、錄制、分析
視頻處理：視頻播放、錄制、編輯
流媒體：實時音視頻、直播推流
WebRTC：點對點通信
媒體格式：編解碼、轉換

音頻處理實現

音頻播放器

// 音頻播放器類
class AudioPlayer {private audio: HTMLAudioElement;private audioContext: AudioContext;private source: MediaElementAudioSourceNode;private analyser: AnalyserNode;private gainNode: GainNode;private equalizer: EqualizerNode;constructor() {this.audio = new Audio();this.audioContext = new AudioContext();// 創建音頻源this.source = this.audioContext.createMediaElementSource(this.audio);// 創建分析器this.analyser = this.audioContext.createAnalyser();this.analyser.fftSize = 2048;// 創建音量控制this.gainNode = this.audioContext.createGain();// 創建均衡器this.equalizer = new EqualizerNode(this.audioContext);// 連接音頻節點this.source.connect(this.analyser).connect(this.equalizer.input).connect(this.gainNode).connect(this.audioContext.destination);// 初始化事件監聽this.initializeEventListeners();}// 加載音頻loadAudio(url: string): Promise<void> {return new Promise((resolve, reject) => {this.audio.src = url;this.audio.load();this.audio.oncanplaythrough = () => resolve();this.audio.onerror = () => reject(new Error('Failed to load audio'));});}// 播放play(): Promise<void> {return this.audio.play();}// 暫停pause(): void {this.audio.pause();}// 跳轉到指定時間seek(time: number): void {this.audio.currentTime = time;}// 設置音量setVolume(volume: number): void {this.gainNode.gain.value = Math.max(0, Math.min(1, volume));}// 設置均衡器setEqualizerBand(frequency: number, gain: number): void {this.equalizer.setBand(frequency, gain);}// 獲取頻譜數據getSpectrumData(): Uint8Array {const dataArray = new Uint8Array(this.analyser.frequencyBinCount);this.analyser.getByteFrequencyData(dataArray);return dataArray;}// 獲取波形數據getWaveformData(): Uint8Array {const dataArray = new Uint8Array(this.analyser.frequencyBinCount);this.analyser.getByteTimeDomainData(dataArray);return dataArray;}// 初始化事件監聽private initializeEventListeners(): void {// 播放狀態變化this.audio.addEventListener('play', () => {this.audioContext.resume();});// 音頻結束this.audio.addEventListener('ended', () => {// 處理播放結束});// 音頻錯誤this.audio.addEventListener('error', (e) => {console.error('Audio error:', e);});}
}// 均衡器節點類
class EqualizerNode {private context: AudioContext;private bands: BiquadFilterNode[];private _input: GainNode;private _output: GainNode;constructor(context: AudioContext) {this.context = context;this.bands = [];// 創建輸入輸出節點this._input = context.createGain();this._output = context.createGain();// 創建均衡器頻段this.createBands();// 連接頻段this.connectBands();}// 獲取輸入節點get input(): AudioNode {return this._input;}// 獲取輸出節點get output(): AudioNode {return this._output;}// 設置頻段增益setBand(frequency: number, gain: number): void {const band = this.bands.find(b => Math.abs(b.frequency.value - frequency) < 1);if (band) {band.gain.value = gain;}}// 創建均衡器頻段private createBands(): void {const frequencies = [60, 170, 310, 600, 1000, 3000, 6000, 12000, 14000, 16000];frequencies.forEach(freq => {const filter = this.context.createBiquadFilter();filter.type = 'peaking';filter.frequency.value = freq;filter.Q.value = 1;filter.gain.value = 0;this.bands.push(filter);});}// 連接頻段private connectBands(): void {this.bands.reduce((prev, curr) => {prev.connect(curr);return curr;}, this._input);this.bands[this.bands.length - 1].connect(this._output);}
}// 使用示例
const player = new AudioPlayer();// 加載并播放音頻
async function playAudio(url: string) {try {await player.loadAudio(url);await player.play();// 設置音量player.setVolume(0.8);// 設置均衡器player.setEqualizerBand(60, 3); // 增強低頻player.setEqualizerBand(12000, 2); // 增強高頻// 實時更新頻譜顯示function updateSpectrum() {const spectrumData = player.getSpectrumData();// 使用頻譜數據繪制可視化效果requestAnimationFrame(updateSpectrum);}updateSpectrum();} catch (error) {console.error('Failed to play audio:', error);}
}

音頻錄制器

// 音頻錄制器類
class AudioRecorder {private stream: MediaStream | null;private mediaRecorder: MediaRecorder | null;private audioChunks: Blob[];private isRecording: boolean;constructor() {this.stream = null;this.mediaRecorder = null;this.audioChunks = [];this.isRecording = false;}// 請求麥克風權限async requestPermission(): Promise<void> {try {this.stream = await navigator.mediaDevices.getUserMedia({audio: true});} catch (error) {throw new Error('Failed to get microphone permission');}}// 開始錄制startRecording(): void {if (!this.stream) {throw new Error('No audio stream available');}this.audioChunks = [];this.mediaRecorder = new MediaRecorder(this.stream);this.mediaRecorder.addEventListener('dataavailable', (event) => {if (event.data.size > 0) {this.audioChunks.push(event.data);}});this.mediaRecorder.start();this.isRecording = true;}// 停止錄制stopRecording(): Promise<Blob> {return new Promise((resolve, reject) => {if (!this.mediaRecorder || !this.isRecording) {reject(new Error('Not recording'));return;}this.mediaRecorder.addEventListener('stop', () => {const audioBlob = new Blob(this.audioChunks, {type: 'audio/webm'});resolve(audioBlob);});this.mediaRecorder.stop();this.isRecording = false;});}// 暫停錄制pauseRecording(): void {if (this.mediaRecorder && this.isRecording) {this.mediaRecorder.pause();}}// 恢復錄制resumeRecording(): void {if (this.mediaRecorder && this.isRecording) {this.mediaRecorder.resume();}}// 釋放資源dispose(): void {if (this.stream) {this.stream.getTracks().forEach(track => track.stop());this.stream = null;}this.mediaRecorder = null;this.audioChunks = [];this.isRecording = false;}
}// 使用示例
const recorder = new AudioRecorder();async function startRecording() {try {// 請求麥克風權限await recorder.requestPermission();// 開始錄制recorder.startRecording();// 5秒后停止錄制setTimeout(async () => {const audioBlob = await recorder.stopRecording();// 創建音頻URLconst audioUrl = URL.createObjectURL(audioBlob);// 創建音頻元素播放錄音const audio = new Audio(audioUrl);audio.play();// 清理資源recorder.dispose();}, 5000);} catch (error) {console.error('Recording failed:', error);}
}

視頻處理實現

視頻播放器

// 視頻播放器類
class VideoPlayer {private video: HTMLVideoElement;private canvas: HTMLCanvasElement;private ctx: CanvasRenderingContext2D;private isPlaying: boolean;constructor(videoElement: HTMLVideoElement,canvas: HTMLCanvasElement) {this.video = videoElement;this.canvas = canvas;this.ctx = canvas.getContext('2d')!;this.isPlaying = false;this.initializePlayer();}// 初始化播放器private initializePlayer(): void {// 設置畫布尺寸this.canvas.width = this.video.clientWidth;this.canvas.height = this.video.clientHeight;// 監聽視頻事件this.video.addEventListener('play', () => {this.isPlaying = true;this.render();});this.video.addEventListener('pause', () => {this.isPlaying = false;});this.video.addEventListener('ended', () => {this.isPlaying = false;});}// 加載視頻loadVideo(url: string): Promise<void> {return new Promise((resolve, reject) => {this.video.src = url;this.video.load();this.video.oncanplaythrough = () => resolve();this.video.onerror = () => reject(new Error('Failed to load video'));});}// 播放play(): Promise<void> {return this.video.play();}// 暫停pause(): void {this.video.pause();}// 跳轉到指定時間seek(time: number): void {this.video.currentTime = time;}// 設置播放速度setPlaybackRate(rate: number): void {this.video.playbackRate = rate;}// 應用濾鏡效果applyFilter(filter: VideoFilter): void {this.ctx.filter = filter.toString();}// 渲染視頻幀private render(): void {if (!this.isPlaying) return;// 繪制視頻幀this.ctx.drawImage(this.video,0,0,this.canvas.width,this.canvas.height);// 繼續渲染下一幀requestAnimationFrame(() => this.render());}// 截取當前幀captureFrame(): string {return this.canvas.toDataURL('image/png');}// 導出視頻片段async exportClip(startTime: number,endTime: number): Promise<Blob> {const stream = this.canvas.captureStream();const recorder = new MediaRecorder(stream);const chunks: Blob[] = [];recorder.ondataavailable = (e) => {if (e.data.size > 0) {chunks.push(e.data);}};return new Promise((resolve, reject) => {recorder.onstop = () => {const blob = new Blob(chunks, { type: 'video/webm' });resolve(blob);};// 開始錄制this.video.currentTime = startTime;recorder.start();// 到達結束時間后停止const checkTime = () => {if (this.video.currentTime >= endTime) {recorder.stop();this.pause();} else {requestAnimationFrame(checkTime);}};this.play().then(checkTime);});}
}// 視頻濾鏡類
class VideoFilter {private filters: Map<string, number>;constructor() {this.filters = new Map();}// 設置亮度setBrightness(value: number): void {this.filters.set('brightness', value);}// 設置對比度setContrast(value: number): void {this.filters.set('contrast', value);}// 設置飽和度setSaturation(value: number): void {this.filters.set('saturate', value);}// 設置色相setHue(value: number): void {this.filters.set('hue-rotate', value);}// 設置模糊setBlur(value: number): void {this.filters.set('blur', value);}// 轉換為CSS濾鏡字符串toString(): string {return Array.from(this.filters.entries()).map(([key, value]) => `${key}(${value}${this.getUnit(key)})`).join(' ');}// 獲取濾鏡單位private getUnit(filter: string): string {switch (filter) {case 'blur':return 'px';case 'hue-rotate':return 'deg';default:return '%';}}
}// 使用示例
const video = document.getElementById('video') as HTMLVideoElement;
const canvas = document.getElementById('canvas') as HTMLCanvasElement;
const player = new VideoPlayer(video, canvas);// 加載并播放視頻
async function playVideo(url: string) {try {await player.loadVideo(url);await player.play();// 應用濾鏡效果const filter = new VideoFilter();filter.setBrightness(110);filter.setContrast(120);filter.setSaturation(130);player.applyFilter(filter);// 截取當前幀setTimeout(() => {const frameData = player.captureFrame();const img = new Image();img.src = frameData;document.body.appendChild(img);}, 3000);// 導出視頻片段setTimeout(async () => {const clip = await player.exportClip(5, 10);const url = URL.createObjectURL(clip);const a = document.createElement('a');a.href = url;a.download = 'clip.webm';a.click();}, 5000);} catch (error) {console.error('Failed to play video:', error);}
}

流媒體處理

WebRTC實現

// WebRTC連接管理器
class WebRTCManager {private peerConnection: RTCPeerConnection;private localStream: MediaStream | null;private remoteStream: MediaStream | null;private dataChannel: RTCDataChannel | null;constructor() {this.peerConnection = new RTCPeerConnection({iceServers: [{ urls: 'stun:stun.l.google.com:19302' }]});this.localStream = null;this.remoteStream = null;this.dataChannel = null;this.initializeConnection();}// 初始化連接private initializeConnection(): void {// 監聽ICE候選this.peerConnection.onicecandidate = (event) => {if (event.candidate) {// 發送ICE候選到遠端this.sendSignalingMessage({type: 'candidate',candidate: event.candidate});}};// 監聽遠端流this.peerConnection.ontrack = (event) => {this.remoteStream = event.streams[0];// 觸發遠端流更新事件this.onRemoteStreamUpdate(this.remoteStream);};// 創建數據通道this.dataChannel = this.peerConnection.createDataChannel('messageChannel');// 監聽數據通道事件this.dataChannel.onmessage = (event) => {this.onDataChannelMessage(event.data);};}// 獲取本地媒體流async getLocalStream(constraints: MediaStreamConstraints): Promise<MediaStream> {try {this.localStream = await navigator.mediaDevices.getUserMedia(constraints);// 添加本地流到連接this.localStream.getTracks().forEach(track => {if (this.localStream) {this.peerConnection.addTrack(track, this.localStream);}});return this.localStream;} catch (error) {throw new Error('Failed to get local stream');}}// 創建連接請求async createOffer(): Promise<RTCSessionDescriptionInit> {try {const offer = await this.peerConnection.createOffer();await this.peerConnection.setLocalDescription(offer);return offer;} catch (error) {throw new Error('Failed to create offer');}}// 創建連接應答async createAnswer(): Promise<RTCSessionDescriptionInit> {try {const answer = await this.peerConnection.createAnswer();await this.peerConnection.setLocalDescription(answer);return answer;} catch (error) {throw new Error('Failed to create answer');}}// 處理遠端描述async handleRemoteDescription(description: RTCSessionDescriptionInit): Promise<void> {try {await this.peerConnection.setRemoteDescription(new RTCSessionDescription(description));if (description.type === 'offer') {const answer = await this.createAnswer();// 發送應答到遠端this.sendSignalingMessage({type: 'answer',answer});}} catch (error) {throw new Error('Failed to handle remote description');}}// 處理ICE候選async handleCandidate(candidate: RTCIceCandidate): Promise<void> {try {await this.peerConnection.addIceCandidate(candidate);} catch (error) {throw new Error('Failed to handle ICE candidate');}}// 發送消息sendMessage(message: string): void {if (this.dataChannel && this.dataChannel.readyState === 'open') {this.dataChannel.send(message);}}// 關閉連接close(): void {if (this.localStream) {this.localStream.getTracks().forEach(track => track.stop());}if (this.dataChannel) {this.dataChannel.close();}this.peerConnection.close();}// 發送信令消息（需要實現）private sendSignalingMessage(message: any): void {// 通過信令服務器發送消息}// 遠端流更新回調（需要實現）private onRemoteStreamUpdate(stream: MediaStream): void {// 處理遠端流更新}// 數據通道消息回調（需要實現）private onDataChannelMessage(message: string): void {// 處理數據通道消息}
}// 使用示例
const rtcManager = new WebRTCManager();// 開始視頻通話
async function startVideoCall() {try {// 獲取本地媒體流const localStream = await rtcManager.getLocalStream({video: true,audio: true});// 顯示本地視頻const localVideo = document.getElementById('localVideo') as HTMLVideoElement;localVideo.srcObject = localStream;// 創建連接請求const offer = await rtcManager.createOffer();// 發送offer到遠端（通過信令服務器）// ...} catch (error) {console.error('Video call failed:', error);}
}// 處理遠端消息
function handleRemoteMessage(message: any) {switch (message.type) {case 'offer':rtcManager.handleRemoteDescription(message.offer);break;case 'answer':rtcManager.handleRemoteDescription(message.answer);break;case 'candidate':rtcManager.handleCandidate(message.candidate);break;}
}