i would change the DB structure slightly...make the catergory field in the books table 1 field, not 3...and store all the values in the one field (like a listing of catergories in one field)
$sql="select from books";
$sql2="select from catergory where catergory in ($cat)";
$result=mysql_query($sql) or die("Couldn't connect");
while ($rows=mysql_fetch_array($result)){
$book = $rows['title'];
$author = $rows['author'];
$isdn = $rows['isdn'];
$cat = $rows['catergory'];
$price = $rows['price'];
$pub = $rows['publisher'];
//above rows not really needed but show for clarity
//show results from 1st query up to catergory
echo "<tr><td>$book</td><td>$author</td><td>$isdn</td>
<td>";
$result2 = mysql_query($sql2) or die ("couldn't connect");
while $rdset2=mysql_fetch_array($result2)){
//creates the second recordset
$catergory=$rdset2['catergory'];
//$mycat is a varaible to hold both the catergories and the
//html to make a nested table
$mycat="<table><tr><td>";
//since the catergories is an array we need to check for
//how many elements to see if we can explode the array
//or just show it
$numrows=count($catergory);
if ($numrows>1){
//more than one value in the field
$cat_list = explode($catergory,",");
foreach ($cat_list as $type){
//concatenate the strings into a little nested table to hold
//the catergories
$mycat=.$type."</td></tr>";
} //end the foreach loop
}else{
//only one value in the field catergory
$mycat =.$catergory."</td></tr>";
}
$mycat=."</table>";
} //end second recordset loop
//continue showing the records including the catergory
echo "$mycat</td><td>$price</td><td>$pub</td></tr>";
} //close 1st recordset
echo "</table>";
// show rest of page-- footers etc
hth