Store the valid data in your div and then compare the image ID with the div's stored data before appending.
HTML
<p>Drag an image into the right Box:</p>
<div>
<div id="div1" ondrop="drop(event)" ondragover="allowDrop(event)" data-allowed="boy"></div>
<div id="div2" ondrop="drop(event)" ondragover="allowDrop(event)" data-allowed="man"></div>
<br>
</div>
<div class="title"><p><span id="left">Boy</span><span id="right">Man</span></p></div>
<br>
<div class="images">
<img id="boy" src="http://images.fitpregnancy.mdpcdn.com/sites/fitpregnancy.com/files/styles/width_360/public/toddler-wearing-headphones_600x600_shutterstock_93821278.jpg" draggable="true" ondragstart="drag(event)" width="100" height="100" onclick="reply_click(this.id)">
<img id="man" src="http://creativestockphoto.com/wp-content/uploads/2014/10/images-of-man-and-woman-in-love.jpg" draggable="true" ondragstart="drag(event)" width="100" height="100" onclick="reply_click(this.id)">
</div>
Javascript
function allowDrop(ev) {
ev.preventDefault();
}
function drag(ev) {
ev.dataTransfer.setData("text", ev.target.id);
}
function reply_click(clicked_id)
{
alert(clicked_id);
return clicked_id;
}
function drop(ev) {
ev.preventDefault();
var data = ev.dataTransfer.getData("text");
if(ev.target.getAttribute("data-allowed")==data)
ev.target.appendChild(document.getElementById(data));
}
View the fiddle here