After some clever work, I found a way to solve the issue by duplicating the OrbitControl Update function and directly setting theta and phi values instead of using incremental updates.
this.setPhiTheta = function(t, p) {
var position = this.object.position;
offset.copy( position ).sub( this.target );
// rotate offset to "y-axis-is-up" space
offset.applyQuaternion( quat );
// angle from z-axis around y-axis
theta = Math.atan2( offset.x, offset.z );
// angle from y-axis
phi = Math.atan2( Math.sqrt( offset.x * offset.x + offset.z * offset.z ), offset.y );
if ( this.autoRotate && state === STATE.NONE ) {
this.rotateLeft( getAutoRotationAngle() );
}
/*theta = 3.105;
phi = 1.48;*/
theta = t;
phi = p
//console.log("theta " + theta)
// restrict theta to be between desired limits
theta = Math.max( this.minAzimuthAngle, Math.min( this.maxAzimuthAngle, theta ) );
// restrict phi to be between desired limits
phi = Math.max( this.minPolarAngle, Math.min( this.maxPolarAngle, phi ) );
// restrict phi to be betwee EPS and PI-EPS
phi = Math.max( EPS, Math.min( Math.PI - EPS, phi ) );
var radius = offset.length() * scale;
// restrict radius to be between desired limits
radius = Math.max( this.minDistance, Math.min( this.maxDistance, radius ) );
// move target to panned location
this.target.add( pan );
offset.x = radius * Math.sin( phi ) * Math.sin( theta );
offset.y = radius * Math.cos( phi );
offset.z = radius * Math.sin( phi ) * Math.cos( theta );
// rotate offset back to "camera-up-vector-is-up" space
offset.applyQuaternion( quatInverse );
position.copy( this.target ).add( offset );
this.object.lookAt( this.target );
//thetaDelta /= 1.2;
//phiDelta /= 1.2;
scale = 1;
pan.set( 0, 0, 0 );
// update condition is:
// min(camera displacement, camera rotation in radians)^2 > EPS
// using small-angle approximation cos(x/2) = 1 - x^2 / 8
if ( lastPosition.distanceToSquared( this.object.position ) > EPS
|| 8 * (1 - lastQuaternion.dot(this.object.quaternion)) > EPS ) {
this.dispatchEvent( changeEvent );
lastPosition.copy( this.object.position );
lastQuaternion.copy (this.object.quaternion );
}
}