مكالمات فيديو بخلفية افتراضية وأدوات مفتوحة المصدر

قراءة بيانات الكاميرا

import cv2
cap = cv2.VideoCapture('/dev/video0')
success, frame = cap.read()

#    720p @ 60 FPS
height, width = 720, 1280
cap.set(cv2.CAP_PROP_FRAME_WIDTH ,width)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT,height)
cap.set(cv2.CAP_PROP_FPS, 60)

while True:
    success, frame = cap.read()

cv2.imwrite("test.jpg", frame)

كشف الخلفية

{
    "name": "bodypix",
    "version": "0.0.1",
    "dependencies": {
        "@tensorflow-models/body-pix": "^2.0.5",
        "@tensorflow/tfjs-node-gpu": "^1.7.1"
    }
}

#  ,   TensorFlow GPU
FROM nvcr.io/nvidia/cuda:10.0-cudnn7-runtime-ubuntu18.04
#  node
RUN apt update && apt install -y curl make build-essential \
    && curl -sL https://deb.nodesource.com/setup_12.x | bash - \
    && apt-get -y install nodejs \
    && mkdir /.npm \
    && chmod 777 /.npm
# ,     
#   tfjs-node-gpu      GPU :(
ENV TF_FORCE_GPU_ALLOW_GROWTH=true
#  node-
WORKDIR /src
COPY package.json /src/
RUN npm install
#      
COPY app.js /src/
ENTRYPOINT node /src/app.js

const tf = require('@tensorflow/tfjs-node-gpu');
const bodyPix = require('@tensorflow-models/body-pix');
const http = require('http');
(async () => {
    const net = await bodyPix.load({
        architecture: 'MobileNetV1',
        outputStride: 16,
        multiplier: 0.75,
        quantBytes: 2,
    });
    const server = http.createServer();
    server.on('request', async (req, res) => {
        var chunks = [];
        req.on('data', (chunk) => {
            chunks.push(chunk);
        });
        req.on('end', async () => {
            const image = tf.node.decodeImage(Buffer.concat(chunks));
            segmentation = await net.segmentPerson(image, {
                flipHorizontal: false,
                internalResolution: 'medium',
                segmentationThreshold: 0.7,
            });
            res.writeHead(200, { 'Content-Type': 'application/octet-stream' });
            res.write(Buffer.from(segmentation.data));
            res.end();
            tf.dispose(image);
        });
    });
    server.listen(9000);
})();

def get_mask(frame, bodypix_url='http://localhost:9000'):
    _, data = cv2.imencode(".jpg", frame)
    r = requests.post(
        url=bodypix_url,
        data=data.tobytes(),
        headers={'Content-Type': 'application/octet-stream'})
    #     numpy-
    #     uint8[width * height]   0  1
    mask = np.frombuffer(r.content, dtype=np.uint8)
    mask = mask.reshape((frame.shape[0], frame.shape[1]))
    return mask

#    (     16:9)
replacement_bg_raw = cv2.imread("background.jpg")

#    ,       (width & height   )
width, height = 720, 1280
replacement_bg = cv2.resize(replacement_bg_raw, (width, height))

#     ,   
inv_mask = 1-mask
for c in range(frame.shape[2]):
    frame[:,:,c] = frame[:,:,c]*mask + replacement_bg[:,:,c]*inv_mask

تجارب مثيرة للاهتمام

def post_process_mask(mask):
    mask = cv2.dilate(mask, np.ones((10,10), np.uint8) , iterations=1)
    mask = cv2.erode(mask, np.ones((10,10), np.uint8) , iterations=1)
    return mask

def post_process_mask(mask):
    mask = cv2.dilate(mask, np.ones((10,10), np.uint8) , iterations=1)
    mask = cv2.blur(mask.astype(float), (30,30))
    return mask

• صورة باهتة أو أحادية اللون - كما لو تم رسمها بواسطة ليزر ساطع.
• تأثير يذكرنا بخطوط المسح أو شيء يشبه الشبكة - كما لو كانت الصورة معروضة بعدة أشعة.
• "تأثير الأشباح" - كما لو أن الإسقاط يتم في طبقات أو كما لو أن المسافة الصحيحة التي يجب عرضها عندها لم يتم الحفاظ عليها أثناء إنشاء الإسقاط.

#     -  
holo = cv2.applyColorMap(frame, cv2.COLORMAP_WINTER)

#    bandLength    10-30%,
#    bandGap.
bandLength, bandGap = 2, 3
for y in range(holo.shape[0]):
    if y % (bandLength+bandGap) < bandLength:
        holo[y,:,:] = holo[y,:,:] * np.random.uniform(0.1, 0.3)

# shift_img : https://stackoverflow.com/a/53140617
def shift_img(img, dx, dy):
    img = np.roll(img, dy, axis=0)
    img = np.roll(img, dx, axis=1)
    if dy>0:
        img[:dy, :] = 0
    elif dy<0:
        img[dy:, :] = 0
    if dx>0:
        img[:, :dx] = 0
    elif dx<0:
        img[:, dx:] = 0
    return img

#    : holo * 0.2 + shifted_holo * 0.8 + 0
holo2 = cv2.addWeighted(holo, 0.2, shift_img(holo1.copy(), 5, 5), 0.8, 0)
holo2 = cv2.addWeighted(holo2, 0.4, shift_img(holo1.copy(), -5, -5), 0.6, 0)

holo_done = cv2.addWeighted(img, 0.5, holo2, 0.6, 0)

إخراج الفيديو

numpy==1.18.2
opencv-python==4.2.0.32
requests==2.23.0
pyfakewebcam==0.1.0

FROM python:3-buster
#   pip
RUN pip install --upgrade pip
#   opencv
RUN apt-get update && \
    apt-get install -y \
      `# opencv requirements` \
      libsm6 libxext6 libxrender-dev \
      `# opencv video opening requirements` \
      libv4l-dev
#    requirements.txt
WORKDIR /src
COPY requirements.txt /src/
RUN pip install --no-cache-dir -r /src/requirements.txt
#   
COPY background.jpg /data/
#     (     )
COPY fake.py /src/
ENTRYPOINT python -u fake.py

sudo apt install v4l2loopback-dkms

sudo modprobe -r v4l2loopback
sudo modprobe v4l2loopback devices=1 video_nr=20 card_label="v4l2loopback" exclusive_caps=1

# ,  ,   ,   ,  width  height
fake = pyfakewebcam.FakeWebcam('/dev/video20', width, height)

frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
fake.schedule_frame(frame)

import os
import cv2
import numpy as np
import requests
import pyfakewebcam

def get_mask(frame, bodypix_url='http://localhost:9000'):
    _, data = cv2.imencode(".jpg", frame)
    r = requests.post(
        url=bodypix_url,
        data=data.tobytes(),
        headers={'Content-Type': 'application/octet-stream'})
    mask = np.frombuffer(r.content, dtype=np.uint8)
    mask = mask.reshape((frame.shape[0], frame.shape[1]))
    return mask

def post_process_mask(mask):
    mask = cv2.dilate(mask, np.ones((10,10), np.uint8) , iterations=1)
    mask = cv2.blur(mask.astype(float), (30,30))
    return mask

def shift_image(img, dx, dy):
    img = np.roll(img, dy, axis=0)
    img = np.roll(img, dx, axis=1)
    if dy>0:
        img[:dy, :] = 0
    elif dy<0:
        img[dy:, :] = 0
    if dx>0:
        img[:, :dx] = 0
    elif dx<0:
        img[:, dx:] = 0
    return img

def hologram_effect(img):
    #    
    holo = cv2.applyColorMap(img, cv2.COLORMAP_WINTER)
    #   
    bandLength, bandGap = 2, 3
    for y in range(holo.shape[0]):
        if y % (bandLength+bandGap) < bandLength:
            holo[y,:,:] = holo[y,:,:] * np.random.uniform(0.1, 0.3)
    #  
    holo_blur = cv2.addWeighted(holo, 0.2, shift_image(holo.copy(), 5, 5), 0.8, 0)
    holo_blur = cv2.addWeighted(holo_blur, 0.4, shift_image(holo.copy(), -5, -5), 0.6, 0)
    #     
    out = cv2.addWeighted(img, 0.5, holo_blur, 0.6, 0)
    return out

def get_frame(cap, background_scaled):
    _, frame = cap.read()
    #      (  ,   )
    #       
    mask = None
    while mask is None:
        try:
            mask = get_mask(frame)
        except requests.RequestException:
            print("mask request failed, retrying")
    # -   
    mask = post_process_mask(mask)
    frame = hologram_effect(frame)
    #     
    inv_mask = 1-mask
    for c in range(frame.shape[2]):
        frame[:,:,c] = frame[:,:,c]*mask + background_scaled[:,:,c]*inv_mask
    return frame

#     
cap = cv2.VideoCapture('/dev/video0')
height, width = 720, 1280
cap.set(cv2.CAP_PROP_FRAME_WIDTH, width)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height)
cap.set(cv2.CAP_PROP_FPS, 60)

#   
fake = pyfakewebcam.FakeWebcam('/dev/video20', width, height)

#    
background = cv2.imread("/data/background.jpg")
background_scaled = cv2.resize(background, (width, height))

#    
while True:
    frame = get_frame(cap, background_scaled)
    #    RGB-
    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    fake.schedule_frame(frame)

docker build -t bodypix ./bodypix
docker build -t fakecam ./fakecam

#  
docker network create --driver bridge fakecam
#   bodypix
docker run -d \
  --name=bodypix \
  --network=fakecam \
  --gpus=all --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 \
  bodypix
#  ,  ,      ,  ,
#           
# ,     `sudo groupadd $USER video`
docker run -d \
  --name=fakecam \
  --network=fakecam \
  -p 8080:8080 \
  -u "$$(id -u):$$(getent group video | cut -d: -f3)" \
  $$(find /dev -name 'video*' -printf "--device %p ") \
  fakecam

ملخص