Pensei algo parecido à resposta do [JuninZe][1]: ter uma função dentro dos frames (`frame_modal()`) que vão simular o Modal Overlay. O `dialog()` abre no frame1 e dispara a função `frame_modal` do frame2 e frame3.

INDEX.HTML

    <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
    <html lang="en">
    <head>  
    </head>
    <frameset rows="175px,*" frameborder="0">
        <frame name="frame1" src="frame1.html">
        <frameset cols="50%,50%" class="child_frameset">
            <frame name="frame2" src="frame2.html">
            <frame name="frame3" src="frame3.html">
        </frameset>
    </frameset>
    </html>

FRAME1.HTML

    <!doctype html>
    <html lang="en">
    <head>
    	<meta charset="utf-8">
    	<title>jQuery UI Dialog - Basic modal</title>
    	<link rel="stylesheet" href="//code.jquery.com/ui/1.10.4/themes/smoothness/jquery-ui.css">
    	<script src="//code.jquery.com/jquery-1.10.2.js"></script>
    	<script src="//code.jquery.com/ui/1.10.4/jquery-ui.js"></script>
    	<link rel="stylesheet" href="//jqueryui.com/resources/demos/style.css">
    	<script>
    	$(function() {
    		function open_mod(){
    			$('.ui-widget-overlay').addClass('custom-overlay');
    			$( ".ui-widget-overlay" ).on('click',function(){
    				$('#dialog-modal').dialog('close');
    			});
    			top.window.frames["frame2"].frame_modal('block');
    			top.window.frames["frame3"].frame_modal('block');
    		}
    		function close_mod(){
    			$('.ui-widget-overlay').removeClass('custom-overlay');
    			top.window.frames["frame2"].frame_modal('none');
    			top.window.frames["frame3"].frame_modal('none');
    		}
    		$('#abre-modal').on('click', function() {
    			$( "#dialog-modal" ).dialog( {
    				height: 140,
    				modal: true,
    				open: open_mod,
    				close: close_mod    
    			} );
    		});
    	});
    	</script>
    	<style>
    	.ui-widget-overlay.custom-overlay
    	{
    		background-color: black;
    		background-image: none;
    		opacity: 0.5;
    	}
    	</style>
    </head>
    <body>
    	<div id="dialog-modal" title="Basic modal dialog">
    		<p>Adding the modal overlay screen makes the dialog look more prominent because it dims out the page content.</p>
    	</div>
    	<a href="javascript:" id="abre-modal">Abrir modal</a></p>
    </body>
    </html>

FRAME2.HTML

    <!DOCTYPE html>  
    <html lang="en">  
    <head>
    	<style>
    	#custom-overlay
    	{
    		background-color: black;
    		background-image: none;
    		opacity: 0.5;
    		position:fixed;
    		left:0;
    		top:0;
    		width:100%;
    		height:100%;
    		display:none;
    		z-index:999;
    	}
    	</style>
    </head>
    <body>
    	<div id="custom-overlay"></div>
    	<h1>I'm frame 2</h1>
    	<script type="text/javascript">
    	function frame_modal(how){
    		document.getElementById("custom-overlay").style.display=how
    	}
    	</script>   
    </body>
    </html>

FRAME3.HTML

 - idêntico ao FRAME2.HTML, só trocando o texto:

        <h1>I'm frame 3</h1>


  [1]: https://pt.stackoverflow.com/a/20907/201