前几天做的终极离谱需求之我是哪个大怨钟，不多逼逼，直接上代码. 先看效果
//package.json中依赖
"dependencies": {
"@mediapipe/selfie_segmentation": "^0.1.1675465747",
"@tensorflow-models/body-segmentation": "^1.0.2",
"@tensorflow/tfjs": "^4.16.0",
"@tensorflow/tfjs-backend-webgl": "^4.16.0",
"@tensorflow/tfjs-converter": "^4.16.0",
"@tensorflow/tfjs-core": "^4.16.0",
"element-plus": "^2.0.2",
"vue": "^3.2.25"
},
.vue文件
<template>
<div style="position: absolute; width: 100%;height: 100%;left: 0;top: 0;z-index: 2000;background-color: aliceblue;overflow: hidden;text-align: center;"
v-loading="params.loading" element-loading-text="图像处理中">
<div style="display: flex;justify-content: center;width: 100%;height:100%; align-items: center;">
<div style="margin-right:100px">
<div style="font-size: 24px;"> 实时图像</div>
<video width="500" height="500" id="video">
</video>
<div style="display:flex;justify-content:center">
<el-button @click="system.saveSouceImage.call(system, true)"
style="font-size: 20px; padding:4px 20px; border: 1px solid #ccc;margin-bottom: 20px; "
type="primary">
卡通化拍摄
</el-button>
<el-button @click="system.saveSouceImage.call(system)"
style="font-size: 20px; padding:4px 20px; border: 1px solid #ccc;margin-bottom: 20px; "
type="primary">
拍摄
</el-button>
<br>
<el-button @click="system.finish.call(system)"
style="font-size: 20px; padding:4px 20px;margin-left:20px;" type="success">
完成
</el-button>
</div>
</div>
<div style="">
<p style="font-size: 24px;margin:0;">合成图片</p>
<canvas id="canvas" width="325" height="587"></canvas>
</div>
</div>
<img src="./1.png" alt="" id="img1" style="object-fit: contain;display: none;">
<img src="./9.png" alt="" id="img9" style="object-fit: contain;display: none;">
</div>
</template>
<script setup>
import { onMounted, reactive, getCurrentInstance, ref } from "vue"
import { generateImage } from './generate.js';
import * as bodySegmentation from '@tensorflow-models/body-segmentation';
import '@tensorflow/tfjs-core';
import '@tensorflow/tfjs-converter';
// Register WebGL backend.
import '@tensorflow/tfjs-backend-webgl';
let params = reactive({
loading: false
})
let system = {
canavs: null,
ctx: null,
sourceImageData: null,
faceImageData: null,
async getUserImage() {
navigator.mediaDevices.getUserMedia({ audio: false, video: true })
.then(stream => {
// 获取到优化后的媒体流
video.srcObject = stream;
video.onloadedmetadata = function (e) {
video.play();
};
}
)
.catch(err => {
ElMessage({
message: '获取摄像头失',
type: 'error',
})
})
},
async saveSouceImage(flag) {
if (!video.srcObject) {
ElMessage.error('请开启摄像头')
this.getUserImage();
return
}
if (!this.canvas) {
this.canvas = document.createElement("canvas");
this.ctx = this.canvas.getContext('2d');
// this.canvas.style='position:absolute;left:0;top:0;z-index:100000'
// document.querySelector('body').appendChild(this.canvas)
}
this.canvas.width = video.videoWidth;
this.canvas.height = video.videoHeight;
this.ctx.drawImage(video, 0, 0, video.videoWidth, video.videoHeight);
if (flag) {
params.loading = true;
await generateImage('m', 1, this.canvas, this.canvas);
params.loading = false;
}
this.sourceImageData = this.ctx.getImageData(0, 0, this.canvas.width, this.canvas.height);
this.fun(this.canvas)
},
async fun(image) {
const segmenter = await bodySegmentation.createSegmenter(bodySegmentation.SupportedModels.BodyPix);
const segmentation = await segmenter.segmentPeople(image, { multiSegmentation: false, segmentBodyParts: true });
let data = segmentation[0].mask.mask.data;
let width = segmentation[0].mask.mask.width;
let height = segmentation[0].mask.mask.width;
let sourceImageData = this.sourceImageData.data
let minX = width;
let minY = height;
let maxX = 0;
let maxY = 0;
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
let i = (y * width + x) * 4;
if (data[i] <= 1) {
minX = Math.min(minX, x);
minY = Math.min(minY, y);
maxX = Math.max(maxX, x);
maxY = Math.max(maxY, y);
} else {
data[i] = 24;
data[i + 3] = 0;
sourceImageData[i] = 0;
sourceImageData[i + 1] = 0;
sourceImageData[i + 2] = 0;
sourceImageData[i + 3] = 0;
}
}
}
this.ctx.putImageData(this.sourceImageData, 0, 0);
this.faceImageData = this.ctx.getImageData(minX, minY, maxX - minX + 1, maxY - minY + 1);
this.canvas.width = maxX - minX + 1;
this.canvas.height = maxY - minY + 1;
this.ctx.putImageData(this.faceImageData, 0, 0)
this.mixImage(minX, minY, maxX - minX + 1, maxY - minY + 1);
},
mixImage_old(dirtyX, dirtyY, dirtyWidth, dirtyHeight) {
let ctx = canvas.getContext('2d');
canvas.width = img1.naturalWidth;
canvas.height = img1.naturalHeight;
ctx.drawImage(img1, 0, 0, img1.naturalWidth, img1.naturalHeight);
// ctx.clearRect(x, y, width, height);
ctx.clearRect(130, 10, 62, 80);
let width = 62
let scale = width / dirtyWidth;
let height = dirtyHeight * scale;
// 159 -31
let centerX = 167;
let bottomY = 104;
ctx.drawImage(this.canvas, centerX - width / 2, bottomY - height, width, height);
// console.log(ctx.getImageData(0, 0, img1.naturalWidth, img1.naturalHeight));
// ctx.putImageData(this.faceImageData, 130, 10, 0, 0, dirtyWidth, dirtyHeight)
// ctx.putImageData(this.faceImageData,0,0)
},
mixImage(dirtyX, dirtyY, dirtyWidth, dirtyHeight) {
let ctx = canvas.getContext('2d');
canvas.width = img9.naturalWidth;
canvas.height = img9.naturalHeight;
// ctx.clearRect(x, y, width, height);
// ctx.clearRect(130, 10, 62, 80);
let width = 140
let scale = width / dirtyWidth;
let height = dirtyHeight * scale;
// 159 -31
let centerX = 240;
let bottomY = 320;
ctx.drawImage(this.canvas, centerX - width / 2, bottomY - height, width, height);
// // ctx.globalCompositeOperation = 'source-in';
ctx.drawImage(img9, 0, 0, img9.naturalWidth, img9.naturalHeight);
// ctx.globalCompositeOperation = 'source-over';
// ctx.putImageData(this.faceImageData, 130, 10, 0, 0, dirtyWidth, dirtyHeight)
// ctx.putImageData(this.faceImageData,0,0)
},
closeVideo() {
let stream = video.srcObject;
const tracks = stream.getTracks();
tracks.forEach(track => track.stop());
video.srcObject = null;
},
finish() {
this.closeVideo();
params.show4 = true;
if (threeSystem.people.material) {
threeSystem.people.material.dispose()
}
const spriteMaterial = new THREE.SpriteMaterial({ map: new THREE.CanvasTexture(canvas) });
threeSystem.people.material = spriteMaterial;
},
downloadImg() {
function getTime() {
const currentDate = new Date();
// 提取年、月、日、时、分、秒
const year = currentDate.getFullYear();
const month = (currentDate.getMonth() + 1).toString().padStart(2, '0'); // 月份是从 0 到 11，需要加 1
const day = currentDate.getDate().toString().padStart(2, '0');
const hours = currentDate.getHours().toString().padStart(2, '0');
const minutes = currentDate.getMinutes().toString().padStart(2, '0');
const seconds = currentDate.getSeconds().toString().padStart(2, '0');
// 构建输出字符串
return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`;
}
if (!this.canvas3) {
this.canvas3 = document.createElement("canvas");
this.ctx3 = this.canvas3.getContext('2d');
}
params.download = false;
threeSystem.renderer.render(threeSystem.scene, threeSystem.camera);
// var dataURL = threeSystem.renderer.domElement.toDataURL("image/png");
// var downloadLink = document.createElement("a");
// downloadLink.href = dataURL;
// downloadLink.download = Math.random() + ".png";
// downloadLink.click();
let { width, height } = threeSystem.renderer.domElement
this.canvas3.width = width;
this.canvas3.height = height;
this.ctx3.clearRect(0, 0, width, height);
this.ctx3.drawImage(threeSystem.renderer.domElement, 0, 0, width, height);
this.ctx3.font = '24px STheiti, SimHei';
this.ctx3.fillText('清华大学', width - 130, height - 60,);
this.ctx3.fillText(getTime(), width - 260, height - 30,);
var dataURL = this.canvas3.toDataURL("image/png");
var downloadLink = document.createElement("a");
downloadLink.href = dataURL;
downloadLink.download = Math.random() + ".png";
downloadLink.click();
},
}
/****************监听动画速度改变****************/
// 页面重置
const { eventBus } = getCurrentInstance().proxy;
eventBus.$on('resetAll', () => {
})
onMounted(() => {
system.getUserImage()
})
window.addEventListener('resize', () => {
})
window.addEventListener('load', () => {
})
</script>
<style lang="less" scoped></style>
generate.js
import * as tf from '@tensorflow/tfjs';
import * as tfc from '@tensorflow/tfjs-converter';
window.tf = tf;
window.tfc = tfc;
window.progress = 0;
window.bytesUsed = 0;
tf.enableProdMode();
let start;
const MODEL_URL = window.location.href + 'model_full/model.json';
function mirrorPadFunc(input, pad_arr) {
return tf.tidy(() => {
for (let i = 0; i < 4; i++) {
if (pad_arr[i][0] !== 0 || pad_arr[i][1] !== 0) {
let slice_size = [-1, -1, -1, -1];
slice_size[i] = pad_arr[i][0];
let slice_begin = [0, 0, 0, 0];
let padding_left = input.slice(slice_begin, slice_size);
slice_size = [-1, -1, -1, -1];
slice_size[i] = pad_arr[i][1];
slice_begin = [0, 0, 0, 0];
slice_begin[i] = input.shape[i] - pad_arr[i][1];
let padding_right = input.slice(slice_begin, slice_size);
input = tf.concat([padding_left, input, padding_right], i);
}
if (pad_arr[i][0] > 1 || pad_arr[i][1] > 1) {
throw new Error("Only input with no more than length one in padding is supported. We have: " + JSON.stringify(pad_arr));
}
}
return input;
});
}
// For debugging purpose:
window.mirrorPadFunc = mirrorPadFunc;
const progressesList = [0.00023367749587460492, 0.054088046653978504, 0.1804816724673639, 0.18052037621199904, 0.2528568019649621, 0.37458444400475477, 0.39315031021211105, 0.39319017797911254, 0.4444196766347441, 0.5207431700988491, 0.550593651422125, 0.5542242372745627, 0.5605664132978859, 0.5806242652109398, 0.5927784050567816, 0.5962346785553008, 0.5981026434950807, 0.5989430676647844, 0.6435568450337933, 0.6676838282371483, 0.6684442258671517, 0.7463103400111626, 0.9019785470675509, 0.95];
let num_called = 0;
const mirrorPad = async (node) => {
let progress = 0.9 * (performance.now() - start)/(15463.61999999499);
/* progressesList.push(progress);
console.log(progressesList); */
if (num_called >= progressesList.length) {
progress = 0.95;
} else {
progress = progressesList[num_called];
}
num_called += 1;
window.progress = progress;
let memoryInfo = tf.memory();
// console.log("Memory Info:", memoryInfo);
window.bytesUsed = memoryInfo.numBytes;
// Use normal pad (not mirror pad):
// return tf.pad(node.inputs[0], node.inputs[1].arraySync(), 0);
await tf.nextFrame();
if (node.attrs.mode !== "reflect") {
throw new Error("Only reflect mode is supported. Mode: " + node.attrs.mode);
}
let pad_tensor = node.inputs[1];
// node.inputs[1].print();
if (node.inputs[0].shape.length === 4) {
let pad_arr = await pad_tensor.array();
let input = node.inputs[0];
return mirrorPadFunc(input, pad_arr);
} else {
throw new Error("Only input of rank 4 is supported. We have: " + JSON.stringify(pad_tensor.arraySync()));
}
};
tfc.registerOp('MirrorPad', mirrorPad);
const generate = async (model, long_side_scale_size, img, output) => {
console.log("Generation start")
let img_tensor = tf.browser.fromPixels(img);
let scaled_img_tensor;
console.log("Original image size:", img_tensor.shape);
if (long_side_scale_size !== -1) {
let scale_factor = (img_tensor.shape[0] > img_tensor.shape[1] ? img_tensor.shape[0] : img_tensor.shape[1]) / long_side_scale_size; // long side scaled size
let scaled_size = [Math.round(img_tensor.shape[0] / scale_factor), Math.round(img_tensor.shape[1] / scale_factor)];
console.log("Scale to:", scaled_size);
scaled_img_tensor = tf.tidy(() => (
tf.image.resizeBilinear(img_tensor, scaled_size).expandDims(0).div(255)
)); // Batch size may be larger
img_tensor.dispose();
} else {
scaled_img_tensor = tf.tidy(() => (
img_tensor.expandDims(0).div(255)
)); // Batch size may be larger
img_tensor.dispose();
}
start = performance.now();
let generated = await model.executeAsync({'test': scaled_img_tensor});
scaled_img_tensor.dispose();
let end = performance.now();
console.log("Image Generated");
console.log(`Took ${(end - start)/1000} s to generate the image`);
await tf.browser.toPixels((generated.squeeze(0).add(1)).div(2), output);
// console.log(generated.print());
generated.dispose();
}
let preHeat = () => {
// Pre-heat
let model_load_start = performance.now();
tfc.loadGraphModel(MODEL_URL).then((model) => {
console.log("Model Loaded");
let model_load_end = performance.now();
console.log(`Took ${(model_load_end - model_load_start)/1000} s to load the model`);
model.dispose();
});
}
let generateImage = async (resize, fp16, img_id, canvas_id) => {
if (fp16) {
// tf.webgl.forceHalfFloat();
tf.env().set('WEBGL_FORCE_F16_TEXTURES', true);
} else {
tf.env().set('WEBGL_FORCE_F16_TEXTURES', false);
}
let long_side_scale_size;
if (resize === "s") {
long_side_scale_size = 100;
} else if (resize === "m") {
long_side_scale_size = 250;
}else if (resize === "l") {
long_side_scale_size = 500;
} else {
long_side_scale_size = -1;
}
let model_load_start = performance.now();
await tfc.loadGraphModel(MODEL_URL).then(async (model) => {
console.log("Model Loaded");
let model_load_end = performance.now();
console.log(`Took ${(model_load_end - model_load_start)/1000} s to load the model`);
await generate(model, long_side_scale_size, img_id, canvas_id);
tf.disposeVariables();
console.log(tf.memory());
});
window.progress = 1.0;
};
export {preHeat, generateImage};
卡通化使用的模型地址 github.com/TonyLianLon…
1.png图片
9.png图片
