Is jQuery an option for you? It makes what you are doing really simple since the code already exists.

http://jqueryui.com/demos/draggable/

Demo

JavaScript Code

window.onload = addListeners;

function addListeners(){
    document.getElementById('dxy').addEventListener('mousedown', mouseDown, false);
    window.addEventListener('mouseup', mouseUp, false);

}

function mouseUp()
{
    window.removeEventListener('mousemove', divMove, true);
}

function mouseDown(e){
  window.addEventListener('mousemove', divMove, true);
}

function divMove(e){
    var div = document.getElementById('dxy');
  div.style.position = 'absolute';
  div.style.top = e.clientY + 'px';
  div.style.left = e.clientX + 'px';
}​

This is a nice no-jQuery script to drag a div: http://jsfiddle.net/g6m5t8co/1/

var mydragg = function() {
  return {
    move: function(divid, xpos, ypos) {
      divid.style.left = xpos + 'px';
      divid.style.top = ypos + 'px';
    },
    startMoving: function(divid, container, evt) {
      evt = evt || window.event;
      var posX = evt.clientX,
        posY = evt.clientY,
        divTop = divid.style.top,
        divLeft = divid.style.left,
        eWi = parseInt(divid.style.width),
        eHe = parseInt(divid.style.height),
        cWi = parseInt(document.getElementById(container).style.width),
        cHe = parseInt(document.getElementById(container).style.height);
      document.getElementById(container).style.cursor = 'move';
      divTop = divTop.replace('px', '');
      divLeft = divLeft.replace('px', '');
      var diffX = posX - divLeft,
        diffY = posY - divTop;
      document.onmousemove = function(evt) {
        evt = evt || window.event;
        var posX = evt.clientX,
          posY = evt.clientY,
          aX = posX - diffX,
          aY = posY - diffY;
        if (aX < 0) aX = 0;
        if (aY < 0) aY = 0;
        if (aX + eWi > cWi) aX = cWi - eWi;
        if (aY + eHe > cHe) aY = cHe - eHe;
        mydragg.move(divid, aX, aY);
      }
    },
    stopMoving: function(container) {
      var a = document.createElement('script');
      document.getElementById(container).style.cursor = 'default';
      document.onmousemove = function() {}
    },
  }
}();
#container {
  position: absolute;
  background-color: blue;
}

#elem {
  position: absolute;
  background-color: green;
  -webkit-user-select: none;
  -moz-user-select: none;
  -o-user-select: none;
  -ms-user-select: none;
  -khtml-user-select: none;
  user-select: none;
}
<div id='container' style="width: 600px;height: 400px;top:50px;left:50px;">
  <div id="elem" onmousedown='mydragg.startMoving(this,"container",event);' onmouseup='mydragg.stopMoving("container");' style="width: 200px;height: 100px;">
    <div style='width:100%;height:100%;padding:10px'>
      <select id=test>
        <option value=1>first
          <option value=2>second
      </select>
      <INPUT TYPE=text value="123">
    </div>
  </div>
</div>

Well, your movement code simplifies to:

div.style.position = "absolute";
div.style.top = e.clientY - (e.clientY - div.offsetTop) + "px";
div.style.left = e.clientX - (e.clientX - div.offsetLeft) + "px";

Basic math here - the e.clientX and e.clientY have absolutely no effect on the position here, so you're just taking the offsetLeft and reassigning it to the style.left, and the same for the top. Thus no movement whatsoever.

What you need to do is save the clientX and clientY when the mousedown happens, and do the subtraction based on that.

Oh and you're also setting the event listener wrong. The way it is now, you have it run divMove once and the return value (undefined here) is the function attached as the listener. Instead, use function(e) {divMove(dxy,e || window.event);}.


I modified Shaedo's code a little bit, wraps it in a function and add a feature that you can drag an element by only parts of it or its children, say the title bar of a div. Note in this demo, you can only drag the red area to move the blue area.

function makeDragable(dragHandle, dragTarget) {
  let dragObj = null; //object to be moved
  let xOffset = 0; //used to prevent dragged object jumping to mouse location
  let yOffset = 0;

  document.querySelector(dragHandle).addEventListener("mousedown", startDrag, true);
  document.querySelector(dragHandle).addEventListener("touchstart", startDrag, true);

  /*sets offset parameters and starts listening for mouse-move*/
  function startDrag(e) {
    e.preventDefault();
    e.stopPropagation();
    dragObj = document.querySelector(dragTarget);
    dragObj.style.position = "absolute";
    let rect = dragObj.getBoundingClientRect();

    if (e.type=="mousedown") {
      xOffset = e.clientX - rect.left; //clientX and getBoundingClientRect() both use viewable area adjusted when scrolling aka 'viewport'
      yOffset = e.clientY - rect.top;
      window.addEventListener('mousemove', dragObject, true);
    } else if(e.type=="touchstart") {
      xOffset = e.targetTouches[0].clientX - rect.left;
      yOffset = e.targetTouches[0].clientY - rect.top;
      window.addEventListener('touchmove', dragObject, true);
    }
  }

  /*Drag object*/
  function dragObject(e) {
    e.preventDefault();
    e.stopPropagation();

    if(dragObj == null) {
      return; // if there is no object being dragged then do nothing
    } else if(e.type=="mousemove") {
      dragObj.style.left = e.clientX-xOffset +"px"; // adjust location of dragged object so doesn't jump to mouse position
      dragObj.style.top = e.clientY-yOffset +"px";
    } else if(e.type=="touchmove") {
      dragObj.style.left = e.targetTouches[0].clientX-xOffset +"px"; // adjust location of dragged object so doesn't jump to mouse position
      dragObj.style.top = e.targetTouches[0].clientY-yOffset +"px";
    }
  }

  /*End dragging*/
  document.onmouseup = function(e) {
    if (dragObj) {
      dragObj = null;
      window.removeEventListener('mousemove', dragObject, true);
      window.removeEventListener('touchmove', dragObject, true);
    }
  }
}

makeDragable('#handle', '#moveable')
#moveable {
    width: 100px;
    height: 100px;
    background: blue;
}

#handle {
    width: 50px;
    height: 50px;
    cursor: move;
    background: red;
}
<div id="moveable">
    <div id="handle">
    </div>
</div>