PWA的力量：具有300行神经网络JS代码的视频监视系统

获取相机和麦克风

this.video = this.querySelector('video')
this.canvas = this.querySelectorAll('canvas')[0]

this.stream = await navigator.mediaDevices.getUserMedia(
   {video: {facingMode: {ideal: "environment"}}, audio: true}
)
this.video.srcObject = this.stream
await new Promise((resolve, reject) => {
   this.video.onloadedmetadata = (_) => resolve()
})
this.W = this.bbox.width = this.canvas.width = this.video.videoWidth
this.H = this.bbox.height = this.canvas.height = this.video.videoHeight

视频截取

this.capture = new ImageCapture(this.stream.getVideoTracks()[0])
this.recorder = new MediaRecorder(this.canvas.captureStream(), {mimeType : "video/webm"})

grab_video()

async function grab_video() {
	this.canvas.drawImage(await this.capture.grabFrame(), 0, 0)
	const img = this.canvas.getImageData(0, 0, this.W, this.H)
	... //    -   img
	... //   -    
        window.requestAnimationFrame(this.grab_video.bind(this))
}

...
async function grab_video() {
	this.canvas.drawImage(this.video, 0, 0)
	...
}

神经网络加载和人工检测

(async () => {
  self.importScripts('https://cdn.jsdelivr.net/npm/@tensorflow/tfjs/dist/tf.min.js')
  self.importScripts('https://cdn.jsdelivr.net/npm/@tensorflow-models/coco-ssd')

  let model = await cocoSsd.load()
  self.postMessage({})

  self.onmessage = async (ev) => {
    const result = await model.detect(ev.data)
    const person = result.find(v => v.class === 'person')
    if (person) 
      self.postMessage({ok: true, bbox: person.bbox})
    else
      self.postMessage({ok: false, bbox: null})
  }
})()

视频录制

this.recorder.rec = new MediaRecorder(this.stream, {mimeType : "video/webm"})
this.recorder.rec.ondataavailable = (ev) => {
   this.chunk = ev.data
   if (this.detected) {
      this.send_chunk()
   } else if (this.recorder.num > 0) {
      this.send_chunk()
      this.recorder.num--
   }
}
...
this.recorder.rec.start()
this.recorder.num = 0
this.recorder.interval = setInterval(() => {
   this.recorder.rec.stop()
   this.recorder.rec.start()
}, CHUNK_DURATION)

Gmail API

await import('https://apis.google.com/js/api.js')
gapi.load('client:auth2', async () => {
   try {
      await gapi.client.init({
         apiKey: API_KEY,
         clientId: CLIENT_ID,
         discoveryDocs: ['https://www.googleapis.com/discovery/v1/apis/gmail/v1/rest'],
         scope: 'https://www.googleapis.com/auth/gmail.send'
      }) 
      if (!gapi.auth2.getAuthInstance().isSignedIn.je) {
         await gapi.auth2.getAuthInstance().signIn()
      }
      this.msg.innerHTML = ''
      this.querySelector('nav').style.display = ''
   } catch(e) {
      this.msg.innerHTML = 'Gmail authorization error: ' + JSON.stringify(e, null, 2)
   }
})

async send_mail(subject, mime_type, body) {
   const headers = {
      'From': '',
      'To': this.email,
      'Subject': 'Balajahe CCTV: ' + subject,
      'Content-Type': mime_type,
      'Content-transfer-encoding': 'base64'
   }
   let head = ''
   for (const [k, v] of Object.entries(headers)) head += k + ': ' + v + '\r\n'
   const request = gapi.client.gmail.users.messages.send({
      'userId': 'me',
      'resource': { 'raw': btoa(head + '\r\n' + body) }
   })
   return new Promise((resolve, reject) => {
      request.execute((res) => {
         if (!res.code) 
            resolve() 
         else 
            reject(res)
      })
   })
}

const a = this.querySelector('a')
URL.revokeObjectURL(a.href)
a.href = URL.createObjectURL(chunk)
a.download = name
a.click()

Web组件世界中的状态管理

button.true {background-color: red}

get detecting() { return this.querySelector('#detecting').className === 'true' }

离线模式

• 在安装事件中-创建高速缓存的新版本，并将所有必需的资源添加到高速缓存中。
• 在Activate事件中-删除除当前版本以外的所有版本的缓存。
• 在fetch事件中-首先，我们尝试从缓存中获取资源，如果找不到资源，我们将发送网络请求，并将其结果添加到缓存中。

尚未解决的问题

1. 在某些手机上，神经网络速度变慢，也许就我而言，COCO-SSD不是最佳选择，但我不是ML专家，因此我采用了第一个听到的声音。
2. 我没有找到一个示例，说明如何通过GAPI发送的视频不是Base64格式，而是原始二进制文件。这将节省处理器时间和网络流量。
3. 我不了解安全性。为了进行本地调试，我将localhost域添加到了Google应用程序中，但是如果有人开始使用应用程序密钥发送垃圾邮件-Google会自行阻止密钥还是发送者的帐户？