讯飞vue3+ts实时语音转文字+录音上传功能

<template>
  <BasicDrawer
    v-bind="$attrs"
    @register="registerDrawer"
    :title="getTitle"
    :width="700"
    @ok="handleSubmit"
    :showFooter="showFooter"
    destroyOnClose
  >
    <a-form
      ref="formRef"
      :model="formModal"
      :label-col="{ span: 4 }"
      :wrapper-col="{ span: 20 }"
    >
      <a-form-item
        label="通信方式"
        name="msgChannel"
        :rules="[{ required: true, message: '请选择通信方式'}]"
      >
        <a-radio-group v-model:value="formModal.msgChannel" @change="onChange">
          <a-radio :value="1">短报文</a-radio>
          <a-radio :value="2">移动通信</a-radio>
        </a-radio-group>
      </a-form-item>
      <a-form-item
        v-if="ifShow"
        label="通播设备"
        name="receiverList"
        :rules="[{ required: true, message: '请选择通播设备' }]"
      >
        <a-tree-select
          v-model:value="formModal.receiverList"
          style="width: 100%"
          tree-checkable
          :show-checked-strategy="SHOW_CHILD"
          :dropdown-style="{ maxHeight: '400px', overflow: 'auto' }"
          :tree-data="treeData"
          :fieldNames="{
            children:'children', label:'name', value: 'id'
          }"
          show-search
          tree-node-filter-prop="name"
          placeholder="请选择通播设备"
        />
      </a-form-item>
      <a-form-item
        label="消息类型"
        name="msgType"
        :rules="[{ required: true, message: '请选择消息类型'}]"
      >
        <a-radio-group v-model:value="formModal.msgType" @change="onChangeType">
          <a-radio :value="1">文字</a-radio>
          <a-radio :value="2">图片</a-radio>
          <a-radio :value="3">语音</a-radio>
        </a-radio-group>
      </a-form-item>
      <a-form-item
        label="消息内容"
        name="msgContent"
        :rules="[{ required: true, message: '请输入消息内容'},{ max:1000, message: '超过最大限制字符' }]"
      >
        <div v-show="formModal.msgType===1">
          <a-textarea :rows="4" v-model:value="formModal.msgContent" placeholder="请输入消息内容"/>
        </div>
        <div v-show="formModal.msgType===2">
          <a-upload
            name="file"
            list-type="picture-card"
            class="avatar-uploader"
            :showUploadList="false"
            accept=".jpg,.jpeg,.gif,.png,.webp"
            :action="uploadUrl"
            :headers="getheader()"
            :before-upload="beforeUpload"
            @change="handleChange"
          >
            <img v-if="imageUrl" :src="imageUrl" alt="avatar"/>
            <div v-else>
              <loading-outlined v-if="loading"></loading-outlined>
              <plus-outlined v-else></plus-outlined>
            </div>
          </a-upload>
        </div>
        <div v-show="formModal.msgType===3">
          <a-textarea v-model:value="formModal.msgContent" disabled="disabled" :rows="4" placeholder="请输入消息内容"/>
          <div class="audioBox">
            <div></div>
            <img v-if="recordStatus===1" @click="onStart" src="@/assets/images/audio.png">
            <div class="ing" v-if="recordStatus===2">
              <img src="@/assets/images/bo.png">
              <span>正在识别中……</span>
            </div>
            <img v-if="recordStatus===2" @click="onFinish" src="@/assets/images/closeaudio.png">
          </div>
        </div>
      </a-form-item>
    </a-form>
  </BasicDrawer>
</template>
<script lang="ts" setup>
import {ref, reactive, computed, onMounted, onUnmounted,watch} from 'vue';
import {BasicDrawer, useDrawerInner} from '/@/components/Drawer';
import {useDrawerAdaptiveWidth} from '/@/hooks/jeecg/useAdaptiveWidth';
import {addSendMsg, terminalList} from './equipmentApi'
import {cloneDeep} from 'lodash-es';
import {PlusOutlined, LoadingOutlined} from '@ant-design/icons-vue';
import {useGlobSetting} from '/@/hooks/setting';
import {getFileAccessHttpUrl, getHeaders} from '/@/utils/common/compUtils';
import {TreeSelect} from 'ant-design-vue';
import {useMessage} from "@/hooks/web/useMessage";
import IatRecorder from "@/utils/xunfei/iatRecorder";// 科大讯飞
import {getUploadFile} from "@/views/system/ossfile/ossfile.api";

/**
 *录音变量
 **/
const recording = ref(false)
const mediaRecorder = ref(null)
const chunks = ref([])
const audioURL = ref('')
const msgContent = ref()
let stream
onMounted(async () => {
  try {
    stream = await navigator.mediaDevices.getUserMedia({audio: true})
    mediaRecorder.value = new MediaRecorder(stream)
    mediaRecorder.value.ondataavailable = (e) => {
      chunks.value.push(e.data)
    }
    mediaRecorder.value.onstop = () => {
      const blob = new Blob(chunks.value, {type: 'audio/mp3; codecs=opus'})
      const file = new File([blob], 'audio.mp3', {type: blob.type})
      chunks.value = []
      audioURL.value = URL.createObjectURL(file)
      let formData = new FormData();
      formData.append('file', file);
      getUploadFile(formData).then((res)=>{
       if(res.data.code===0){
         msgContent.value = res.data.message
       }
      })
    }
  } catch (error) {
    console.error('获取音频输入设备失败：', error)
  }
})

/**
 * 图片上传
 */
const imageUrl = ref()
const loading = ref(false)
const {domainUrl} = useGlobSetting();
const uploadUrl = domainUrl + '/sys/common/upload';
const getheader = () => {
  return getHeaders();
}

const getBizData = () => {
  return {
    biz: 'jeditor',
    jeditor: '1',
  };
}
const {createMessage: $message} = useMessage();
const beforeUpload = (file) => {
  const isJpgOrPng = file.type === 'image/jpeg' || file.type === 'image/png';
  if (!isJpgOrPng) {
    $message.error('请上传正确的格式');
  }
  return isJpgOrPng;
};

const handleChange = (info: Recordable) => {
  const file = info.file;
  const status = file?.status;

  if (status === 'uploading') {
    if (!loading.value) {
      loading.value = true;
    }
  } else if (status === 'done') {
    let realUrl = getFileAccessHttpUrl(file.response.message);
    imageUrl.value = realUrl
    formModal.msgContent = realUrl
    loading.value = false;
  } else if (status === 'error') {
    loading.value = false;
  }
}

const onChangeType = () => {
  formModal.msgContent = ''
}




/**
 * 语音转文字
 */

let iatRecorder = new IatRecorder();


// 状态改变时处罚
iatRecorder.onWillStatusChange = function (oldStatus, status) {
  if (status === "ing") {
    recordStatus.value = 2
  }
  if (status === "end") {
    recordStatus.value = 1
  }
};

const recordStatus = ref(1)
watch(recordStatus,(newValue, oldValue)=>{
   if(newValue===1){
     console.log(newValue)
     mediaRecorder.value.stop()
   }
})

// 监听识别结果的变化
iatRecorder.onTextChange = function (text) {
  formModal.msgContent = text
};

const onStart = () => {
  iatRecorder.start();
  mediaRecorder.value.start()
};

const onFinish = () => {
  iatRecorder.stop();
  mediaRecorder.value.stop()
}

const closeStream = (stream) => {
  if (stream) {
    stream.getTracks().forEach((track) => track.stop())
  }
}

onUnmounted(() => {
  closeStream(stream)
})

const ifShow = ref(true)
const record = ref()
const getTitle = computed(() => {
  return ifShow.value ? '通播发送' : '点播发送'
})

const showFooter = ref(true);

const formModal = reactive({
  msgChannel: 1,
  msgContent: "",
  msgType: 1,
  receiverList: []
})


// 声明Emits
const emit = defineEmits(['success', 'register']);
const [registerDrawer, {setDrawerProps, closeDrawer}] = useDrawerInner(async (data) => {
  const obj = {
    msgChannel: 1,
    msgContent: "",
    msgType: 1,
    receiverList: []
  }
  Object.assign(formModal, obj)
  ifShow.value = data.ifShow
  record.value = data.record
  if (ifShow.value) {
    getTerminalList()
  }
})

const SHOW_CHILD = TreeSelect.SHOW_CHILD;

const treeData = ref()
//重置表单
const resetForm = () => {
  const newData = {
    msgChannel: 1,
    msgContent: "",
    msgType: "",
    receiverList: []
  }
  Object.assign(formModal, newData);
}

const onChange = () => {
  formModal.receiverList = []
  getTerminalList()
}

const getTerminalList = () => {
  terminalList({msgChannel: formModal.msgChannel}).then((res) => {
    if (res) {
      treeData.value = res
    }
  })
}
const {adaptiveWidth} = useDrawerAdaptiveWidth();
const formRef = ref()

async function handleSubmit() {
  const params = cloneDeep(formModal)
  if (!ifShow.value) {
    if (formModal.msgChannel === 1) {
      if (record.value.bdCode) {
        params.receiverList.push(record.value.bdCode)
      } else {
        $message.error('北斗卡号不存在')
        return
      }
    } else {
      if (record.value.terminalId) {
        params.receiverList.push(record.value.terminalId)
      }
    }
  }

  try {
    await formRef.value.validate().then(() => {
      const newData = {...params}
      if (params.msgType === 1) {
        newData.msgType = 0
      } else if (params.msgType === 2) {
        newData.msgType = 3
      } else {
        newData.msgContent = msgContent.value
        newData.msgType = 1
      }
      addSendMsg(newData).then(() => {
        resetForm()
        closeDrawer();
        //刷新列表
        emit('success');
      })

    })
  } catch (error) {
  }
}
</script>
<style lang="less" scoped>
.audioPlay {
  display: flex;
  align-items: center;
  margin-top: 20px
}

.audioBox {
  cursor: pointer;
  display: flex;
  align-items: top;
  justify-content: space-between;
  padding: 10px 0;

  img {
    width: 30px;
    height: 30px;
  }

  .ing {
    display: flex;
    flex-direction: column;
    align-items: center;

    span {
      margin-top: 10px;
    }
  }
}
</style>

封装讯飞组件

import CryptoES from "crypto-es"; // 科大讯飞
const transWorker = new Worker(
 new URL("@/utils/xunfei/transcode.worker.js", import.meta.url)
);

/**
 * 获取websocket url
 * 该接口需要后端提供，这里为了方便前端处理
//  */
const APPID = "5c9c7714";
const API_SECRET = "OTkzMzcyN2NkODFiZmZhMTE2NmQzNjc2";
const API_KEY = "03458eaaa4ffe7c1c72d5be87f98267c";


const getWebSocketUrl = () => {
    return new Promise((resolve, reject) => {
        // 请求地址根据语种不同变化
        var url = "wss://iat-api.xfyun.cn/v2/iat";
        var host = "iat-api.xfyun.cn";
        var apiKey = API_KEY;
        var apiSecret = API_SECRET;
        var date = new Date().toGMTString();
        var algorithm = "hmac-sha256";
        var headers = "host date request-line";
        var signatureOrigin = `host: ${host}\ndate: ${date}\nGET /v2/iat HTTP/1.1`;
        var signatureSha = CryptoES.HmacSHA256(signatureOrigin, apiSecret);
        var signature = CryptoES.enc.Base64.stringify(signatureSha);
        var authorizationOrigin = `api_key="${apiKey}", algorithm="${algorithm}", headers="${headers}", signature="${signature}"`;
        var authorization = btoa(authorizationOrigin);
        url = `${url}?authorization=${authorization}&date=${date}&host=${host}`;
        resolve(url);
    });
};
class IatRecorder {
 public status: string;
    public accent: string;
    public language: string;
    public appId: string;
    public audioData: any[];
    public resultText: string;
    public resultTextTemp: string;
    public onWillStatusChange?: (
        prevStatus: string,
        nextStatus: string
    ) => void;
    public onTextChange?: (text: string) => void;
    constructor(
        { language, accent, appId } = {} as {
            language?: string;
            accent?: string;
            appId?: string;
        }
    ) {
        let self = this;
        this.status = "null";
        this.language = language || "zh_cn";
        this.accent = accent || "mandarin";
        this.appId = appId || APPID;
        // 记录音频数据
        this.audioData = [];
        // 记录听写结果
        this.resultText = "";
        // wpgs下的听写结果需要中间状态辅助记录
        this.resultTextTemp = "";
        transWorker.onmessage = function (event) {
            self.audioData.push(...event.data);
        };
    }
 // 修改录音听写状态
 setStatus(status:string) {
   this.onWillStatusChange && this.status !== status && this.onWillStatusChange(this.status, status)
   this.status = status
 }
 setResultText({ resultText, resultTextTemp }:{ resultText?: string; resultTextTemp?: string } = {}) {
   this.onTextChange && this.onTextChange(resultTextTemp || resultText || '')
   resultText !== undefined && (this.resultText = resultText)
   resultTextTemp !== undefined && (this.resultTextTemp = resultTextTemp)
 }
 // 修改听写参数
 setParams({ language, accent }:{language?:string; accent?:string} = {}) {
   language && (this.language = language)
   accent && (this.accent = accent)
 }
 // 连接websocket
 connectWebSocket() {
   return getWebSocketUrl().then(url => {
  let iatWS
  if ('WebSocket' in window) {
    iatWS = new WebSocket(url)
  } else if ('MozWebSocket' in window) {
    iatWS = new MozWebSocket(url)
  } else {
    alert('浏览器不支持WebSocket')
    return
  }
  this.webSocket = iatWS
  this.setStatus('init')
  iatWS.onopen = e => {
    this.setStatus('ing')
    // 重新开始录音
    setTimeout(() => {
   this.webSocketSend()
    }, 500)
  }
  iatWS.onmessage = e => {
    this.result(e.data)
  }
  iatWS.onerror = e => {
    console.log(`${e.code}`,'onerroronerroronerror')
    this.recorderStop()
  }
  iatWS.onclose = e => {
    console.log(`${e.code}`,'oncloseonclose')
    this.recorderStop()
  }
   })
 }
 // 初始化浏览器录音
 recorderInit() {
   navigator.getUserMedia =
  navigator.getUserMedia ||
  navigator.webkitGetUserMedia ||
  navigator.mozGetUserMedia ||
  navigator.msGetUserMedia

   // 创建音频环境
   try {
  this.audioContext = new (window.AudioContext || window.webkitAudioContext)()
  this.audioContext.resume()
  if (!this.audioContext) {
    alert('浏览器不支持webAudioApi相关接口')
    return
  }
   } catch (e) {
  if (!this.audioContext) {
    alert('浏览器不支持webAudioApi相关接口')
    return
  }
   }

   // 获取浏览器录音权限
   if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
  navigator.mediaDevices
    .getUserMedia({
   audio: true,
   video: false,
    })
    .then(stream => {
   getMediaSuccess(stream)
    })
    .catch(e => {
   getMediaFail(e)
    })
   } else if (navigator.getUserMedia) {
  navigator.getUserMedia(
    {
   audio: true,
   video: false,
    },
    stream => {
   getMediaSuccess(stream)
    },
    function (e) {
   getMediaFail(e)
    }
  )
   } else {
  if (navigator.userAgent.toLowerCase().match(/chrome/) && location.origin.indexOf('https://') < 0) {
    alert('chrome下获取浏览器录音功能，因为安全性问题，需要在localhost或127.0.0.1或https下才能获取权限')
  } else {
    alert('无法获取浏览器录音功能，请升级浏览器或使用chrome')
  }
  this.audioContext && this.audioContext.close()
  return
   }
   // 获取浏览器录音权限成功的回调
   let getMediaSuccess = stream => {
  console.log('getMediaSuccess')
  // 创建一个用于通过JavaScript直接处理音频
  this.scriptProcessor = this.audioContext.createScriptProcessor(0, 1, 1)
  this.scriptProcessor.onaudioprocess = e => {
    // 去处理音频数据
    if (this.status === 'ing') {
   transWorker.postMessage(e.inputBuffer.getChannelData(0))
    }
  }
  // 创建一个新的MediaStreamAudioSourceNode 对象，使来自MediaStream的音频可以被播放和操作
  this.mediaSource = this.audioContext.createMediaStreamSource(stream)
  // 连接
  this.mediaSource.connect(this.scriptProcessor)
  this.scriptProcessor.connect(this.audioContext.destination)
  this.connectWebSocket()
   }

   let getMediaFail = (e) => {
  alert('请求麦克风失败')
  console.log(e)
  this.audioContext && this.audioContext.close()
  this.audioContext = undefined
  // 关闭websocket
  if (this.webSocket && this.webSocket.readyState === 1) {
    this.webSocket.close()
  }
   }
 }
 recorderStart() {
   if (!this.audioContext) {
  this.recorderInit()
   } else {
  this.audioContext.resume()
  this.connectWebSocket()
   }
 }
 // 暂停录音
 recorderStop() {
   // safari下suspend后再次resume录音内容将是空白，设置safari下不做suspend
   if (!(/Safari/.test(navigator.userAgent) && !/Chrome/.test(navigator.userAgen))) {
  this.audioContext && this.audioContext.suspend()
   }
   this.setStatus('end')
 }
 // 处理音频数据
 // transAudioData(audioData) {
 //   audioData = transAudioData.transaction(audioData)
 //   this.audioData.push(...audioData)
 // }
 // 对处理后的音频数据进行base64编码，
 toBase64(buffer) {
   var binary = ''
   var bytes = new Uint8Array(buffer)
   var len = bytes.byteLength
   for (var i = 0; i < len; i++) {
  binary += String.fromCharCode(bytes[i])
   }
   return window.btoa(binary)
 }
 // 向webSocket发送数据
 webSocketSend() {
   if (this.webSocket.readyState !== 1) {
  return
   }
   let audioData = this.audioData.splice(0, 1280)
   var params = {
  common: {
    app_id: this.appId,
  },
  business: {
    language: this.language, //小语种可在控制台--语音听写（流式）--方言/语种处添加试用
    domain: 'iat',
    accent: this.accent, //中文方言可在控制台--语音听写（流式）--方言/语种处添加试用
    vad_eos: 5000,
    dwa: 'wpgs', //为使该功能生效，需到控制台开通动态修正功能（该功能免费）
  },
  data: {
    status: 0,
    format: 'audio/L16;rate=16000',
    encoding: 'raw',
    audio: this.toBase64(audioData),
  },
   }
   console.log(audioData, 'audioData')
   this.webSocket.send(JSON.stringify(params))
   this.handlerInterval = setInterval(() => {
  // websocket未连接
  if (this.webSocket.readyState !== 1) {
    this.audioData = []
    clearInterval(this.handlerInterval)
    return
  }
  if (this.audioData.length === 0) {
    if (this.status === 'end') {
   this.webSocket.send(
     JSON.stringify({
    data: {
      status: 2,
      format: 'audio/L16;rate=16000',
      encoding: 'raw',
      audio: '',
    },
     })
   )
   this.audioData = []
   clearInterval(this.handlerInterval)
    }
    return false
  }
  audioData = this.audioData.splice(0, 1280)
  // 中间帧
  this.webSocket.send(
    JSON.stringify({
   data: {
     status: 1,
     format: 'audio/L16;rate=16000',
     encoding: 'raw',
     audio: this.toBase64(audioData),
   },
    })
  )
   }, 40)
 }
 result(resultData) {
   // 识别结束
   let jsonData = JSON.parse(resultData)
   if (jsonData.data && jsonData.data.result) {
  let data = jsonData.data.result
  let str = ''
  let resultStr = ''
  let ws = data.ws
  for (let i = 0; i < ws.length; i++) {
    str = str + ws[i].cw[0].w
  }
  // 开启wpgs会有此字段(前提：在控制台开通动态修正功能)
  // 取值为 "apd"时表示该片结果是追加到前面的最终结果；取值为"rpl" 时表示替换前面的部分结果，替换范围为rg字段
  if (data.pgs) {
    if (data.pgs === 'apd') {
   // 将resultTextTemp同步给resultText
   this.setResultText({
     resultText: this.resultTextTemp,
   })
    }
    // 将结果存储在resultTextTemp中
    this.setResultText({
   resultTextTemp: this.resultText + str,
    })
  } else {
    this.setResultText({
   resultText: this.resultText + str,
    })
  }
   }
   if (jsonData.code === 0 && jsonData.data.status === 2) {
  this.webSocket.close()
   }
   if (jsonData.code !== 0) {
  this.webSocket.close()
  console.log(`${jsonData.code}:${jsonData.message}`)
   }
 }
 start() {
   this.recorderStart()
   this.setResultText({ resultText: '', resultTextTemp: '' })
 }
 stop() {
   this.recorderStop()
 }
  }
export default IatRecorder

修改调用web 录音功能

const recording = ref(false)
const mediaRecorder = ref(null)
const recordStatus = ref(1)
const chunks = ref([])
const audioURL = ref('')
let stream
let silenceTimeout = null;
const silenceThreshold = 0.01; // 静音阈值，可以根据需要调整
const silenceDuration = 3000; // 静音持续时间 (毫秒)


onMounted(async () => {
  try {
    const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
    const audioContext = new AudioContext();
    const source = audioContext.createMediaStreamSource(stream);
    const analyser = audioContext.createAnalyser();
    analyser.fftSize = 2048;
    const dataArray = new Uint8Array(analyser.fftSize);
    source.connect(analyser);

    mediaRecorder.value = new MediaRecorder(stream);
    mediaRecorder.value.ondataavailable = (e) => {
      chunks.value.push(e.data);
    };

    mediaRecorder.value.onstop = () => {
      const blob = new Blob(chunks.value, { type: 'audio/mp3; codecs=opus' });
      const file = new File([blob], 'audio.mp3', { type: blob.type });
      chunks.value = [];
      audioURL.value = URL.createObjectURL(file);
      let formData = new FormData();
      formData.append('file', file);
      getUploadFile(formData).then((res) => {
        if (res.data.code === 0) {
          formModal.msgContent = res.data.message;
        }
      });
    };

    function checkSilence() {
      analyser.getByteTimeDomainData(dataArray);
      let maxVal = 0;
      for (const val of dataArray) {
        maxVal = Math.max(maxVal, (val - 128) / 128);
      }
      if (maxVal < silenceThreshold) {
        if (!silenceTimeout) {
          silenceTimeout = setTimeout(() => {
            if (mediaRecorder.value && mediaRecorder.value.state === 'recording') {
              onFinish()
              stream.getTracks().forEach((track) => track.stop());
            }
          }, silenceDuration);
        }
      } else {
        clearTimeout(silenceTimeout);
        silenceTimeout = null;
      }
      requestAnimationFrame(checkSilence);
    }

    checkSilence();
  } catch (error) {
    console.error('获取音频输入设备失败：', error);
  }
});

const onStart = () => {
  formModal.msgContent = "";
  mediaRecorder.value.start();
  recordStatus.value = 2;
};

const onFinish = () => {
  mediaRecorder.value.stop();
  recordStatus.value = 1;
};