use mysql_num_rows($result) when you query your DB. Do this first before executing the insert query.
$sql= "select * from usertable where username = $username";
$result = mysql_query($sql,$con);
if (mysql_num_rows($result)){
echo "username already exists. please choose another";
}else{
$sql2 = "insert into usertable values('$username', '$password')";
$result2= mysql_query($sql2)
}
hth
bastien