I am trying to achieve an interesting effect by overlaying a html iframe on top of an augmented reality marker. However, I seem to be facing some issues with the CSS3DRenderer not producing the same result as the WebGLRenderer and I'm struggling to pinpoint where I might be making a mistake.
While the WebGLRenderer works perfectly, smoothly rendering the mesh following the marker, the CSS3DRenderer seems to position the iframe at the center of the video, inverted and unscaled, rotating in the opposite direction.
Thanks to the amazing capabilities of three.js and artoolkit, I have put together some test code using video input and both renderers.
new Promise(function(resolve,reject) {
var source = document.createElement('video');
source.autoplay = true;
source.playsinline = true;
source.controls = false;
source.loop = true;
source.onplay = function(event) {
resolve(source);
}
source.src = 'data/output_4.ogg';
document.body.appendChild(source);
}).then(function(source) {
var scene = new THREE.Scene();
var camera = new THREE.Camera();
camera.matrixAutoUpdate = false;
scene.add(camera);
var material = new THREE.MeshNormalMaterial({
transparent : true,
opacity : 0.5,
side : THREE.DoubleSide
});
var geometry = new THREE.PlaneGeometry(1,1);
var mesh = new THREE.Mesh(geometry,material);
// mesh.matrixAutoUpdate = false;
scene.add(mesh);
var renderer = new THREE.WebGLRenderer({
antialias : true,
alpha : true
});
renderer.setSize(source.videoWidth,source.videoHeight);
renderer.setClearColor(new THREE.Color('lightgrey'),0);
document.body.appendChild(renderer.domElement);
/*\
cssRenderer
\*/
var cssRenderer = new THREE.CSS3DRenderer();
cssRenderer.setSize(source.videoWidth,source.videoHeight);
var cssScene = new THREE.Scene();
var iframe = document.createElement("iframe");
iframe.src = "/data/index.html";
iframe.style.background = "rgb(0,0,0)";
var iframe3D = new THREE.CSS3DObject(iframe);
// iframe3D.matrixAutoUpdate = false;
cssScene.add(iframe3D);
document.body.appendChild(cssRenderer.domElement);
/*\
arController
\*/
var cameraParameters = new ARCameraParam();
var arController = null;
cameraParameters.onload = function() {
arController = new ARController(source.videoWidth,source.videoHeight,cameraParameters);
arController.addEventListener("getMarker",function(event) {
var modelViewMatrix = new THREE.Matrix4().fromArray(event.data.matrix);
camera.matrix.getInverse(modelViewMatrix);
// mesh.matrix.copy(modelViewMatrix);
// iframe3D.matrix.copy(modelViewMatrix);
});
var cameraViewMatrix = new THREE.Matrix4().fromArray(arController.getCameraMatrix());
camera.projectionMatrix.copy(cameraViewMatrix);
}
cameraParameters.load("data/camera_para.dat");
/*\
animate
\*/
requestAnimationFrame(function animate() {
requestAnimationFrame(animate);
if (!arController) {
return;
}
arController.process(source);
renderer.render(scene,camera);
cssRenderer.render(cssScene,camera);
});
});
I had thought that adjusting the camera instead of the object might help solve the issue, but it seems like there could be a missing matrix transformation that needs to be applied.