SELECT AVG( valor ) , HOUR( `timestamp` )
FROM tabela
WHERE DATE_SUB( `timestamp` , INTERVAL 1 HOUR )
GROUP BY HOUR( `timestamp` )
Resultado
Para pegar as médias por hora de todos os dias
SELECT AVG( valor ) , HOUR( `timestamp` ),day( `timestamp` )
FROM tabela
WHERE DATE_SUB( `timestamp` , INTERVAL 1 HOUR )
GROUP BY HOUR( `timestamp` ), day( `timestamp` )