CCSM/ESMF Stage-1 meeting

Time in Driver Run-Loop

2:45 pm MST, Damon Room

High level run driver overview – some alturnatives, future flexibility...
Agenda:

Questions on last meeting:
- Is one subroutine for initialization what we want? – Yes, move all of initialization into that one subroutine.
- Is passing time-info down to components ok? (Note if pass up – must have an additional initialization phase) – For Stage-1 we'll do this at Stage-2 we need to re-evaluate keeping in mind that we'll have to make changes to component models.
- Note that must change timing for when couplers are called.
- Here's the link to last meeting Feb 24th
Questions brought up on Component Rules email:
- Current CCSM components time-stamp data differently than own clock – maybe a time-step off or so...
- Calling ESMF Components is more expensive than a normal subroutine call – how about we add integer date, and time-of-day as an attribute to ESMF State data?
- Currently clocks sent to components in ESMF are read-only. Only way to pass info up – is via clock in component derived type. Or you could add data to the export state with meta-data to describe it – see above suggestion.
Subtle time problem – export states not valid till alarm rings again..
Concurrent case – states are prepared for different times..
Look at top level run driver
Examine some alturnatives..

Next meeting in 2-weeks – look at pseudo-code for driver with mrg components doing time-averaging.

Slightly modified from last time

Here we move the coupler calls that act on component export states to before the component runs rather than after. The reason for that is that the export state after the component runs – represents the state at the next time the component will run – not the current time.
This has the advantage that the time-averaging complexity inherient in one model (only CAM, not other data atm models) is hidden inside of CAM and not elsewhere – this makes plug-and-play easier.

 !
 ! Time loop
 !
do while( .not. ESMF_ClockIsStopTime( ESyncClock )  )
   !
  ! Averaging rules   components are responsible for any averaging done on within 
  ! their coupling interval. Merger will average atmosphere fields for surface components
  ! and put the average on the given surface component import state. Merger will also average
  ! surface data to the atmosphere model coupling interval and put it on the atm import
  !  state
  !
   if ( shr_inputinfo_RunModel( initinfo,  atm  ) .and. ESMF_AlarmIsRinging( atm_alarm ) )then
        ! Run coupler on atmosphere export state before running...
        call ESMF_CplCompRun ( cc_map_a2l, import=a2x_a, export=a2x_l, rc=rc )
        call ESMF_CplCompRun ( cc_map_a2i, import=a2x_a, export=a2x_i, rc=rc )
        call ESMF_CplCompRun ( cc_map_a2o, import=a2x_a, export=a2x_o, rc=rc )
        ! Now run an atmosphere time-step -- don't use export state until reach next time to run
        call ESMF_GridCompRun( gc_atm,     import=x2a_a, export=a2x_a, ESyncClock, phase=1 )
   end if
   if ( shr_inputinfo_RunModel( initinfo,  lnd  ) .and. ESMF_AlarmIsRinging( lnd_alarm ) )then
       call ESMF_CplCompRun (  cc_map_l2a, import=l2x_l, export=l2x_a, rc=rc )
       ! Note: cs_l is the composite state pointing to: a2x_l
       call ESMF_GridCompRun ( gc_mrg_x2l, import=cs_l, export=x2l_l, rc=rc ) 
       call ESMF_GridCompRun(	 gc_lnd,     import=x2l_l, export=l2x_l, ESyncClock, phase=1 )
       call ESMF_GridCompRun(	 gc_lnd,	                                      phase=2 )
       call ESMF_AlarmRingerOff( lnd_alarm )
   end if
   if ( shr_inputinfo_RunModel( initinfo,  ice  ) .and. ESMF_AlarmIsRinging( ice_alarm ) )then
        call ESMF_CplCompRun (  cc_map_i2a, import=i2x_i, export=i2x_a, rc=rc )
        call ESMF_CplCompRun (  cc_map_i2o, import=i2o_i, export=i2o_o, rc=rc )
        ! Note: cs_i is the composite state pointing to: a2x_i, o2i_i
        call ESMF_GridCompRun ( gc_mrg_x2i, import=cs_i, export=x2i_i, rc=rc )
        call ESMF_GridCompRun(  gc_ice,     import=x2i_i, export=i2x_i, ESyncClock, phase=1 )
        call ESMF_GridCompRun(  gc_ice,	                                       phase=2 )
        call ESMF_AlarmRingerOff( ice_alarm )
   end if
   if ( shr_inputinfo_RunModel( initinfo,  ocn  ) .and. ESMF_AlarmIsRinging( ocn_alarm )  )then
        call ESMF_CplCompRun(   cc_map_o2a, import=o2x_o, export=o2x_a, rc=rc)
        call ESMF_CplCompRun (  cc_map_o2i, import=o2x_o, export=o2x_i, rc=rc )
        ! Note: cs_o is the composite state pointing to: i2o_o and a2x_o
        call ESMF_GridCompRun ( gc_mrg_x2o, import=cs_o, export=x2o_o, rc=rc ) 
        call ESMF_GridCompRun(  gc_ocn,     import=x2o_o, export=o2x_o, ESyncClock, phase=1 )
        call ESMF_AlarmRingerOff( ocn_alarm )
   end if
   if ( shr_inputinfo_RunModel( initinfo,  atm  ) .and. ESMF_AlarmIsRinging( atm_alarm ) )then
       ! Note: cs_a is the composite state pointing to: l2x_a, i2x_a, and o2x_a
       call ESMF_GridCompRun( gc_mrg_x2a, import=cs_a, export=x2a_a, rc=rc )
       call ESMF_GridCompRun(	gc_atm,     import=x2a_a,               phase=2 )
       call ESMF_AlarmRingerOff( atm_alarm )
   end if
   ! Advance the clock
   call ESMF_AdvanceClock( ESyncClock )
   ! Write restart information out
   if ( ESMF_AlarmIsRinging( restart ) )then
      call shr_inputinfo_WriteRPointer(  ymd, tod, mpicom, masterproc, initinfo, rest_file )
      call shr_inputinfo_WriteRestart(    rest_file, mpicom, masterproc, initinfo  )
      call shr_timemgr_WriteRestart(  rest_file, mpicom, masterproc, SyncClock )
      call shr_inputinfo_ArchiveRestart( initinfo, masterproc )
      call ESMF_AlarmRingerOff( restart )
   end if
end do

What atm_comp_esmf would look like for cam...

atm_comp_run1:

   if ( sync_clock%curr_time /= my_clock%curr_time ) abort
   if ( sync_clock%atm_alarm /= ringing ) abort
   call ESMF_AlarmGet( sync_clock%atm_alarm, next_ring_time=next_time )
   next_time = next_time - my_clock%TimeStep
   ! Zero out time-average
   call eshr_EState_TimeAvgZero( a2x_a, n )
   ! Convert ESMF import state to CAM internal input data-type
   call atm_Import_ESMF( x2a_a, cam_in )
   do while ( (.not. ESMF_ClockIsStopTime(my_clock)) .and. (my_clock%curr_time .lt. next_time) )
      call cam_run1( cam_in, cam_out )
      if ( my_clock%curr_time .lt. next_time )then
        call cam_run2( cam_out, cam_in )
        call cam_run3( cam_out )
        call cam_run4( cam_out, cam_in, rstwr, nlend )
        call advance_clock( my_clock )
      end if
      ! Convert CAM internal output data-type to ESMF export state
      call atm_Export_ESMF( cam_out, atm_export )
      ! Add to Time-average
      call eshr_EState_TimeAvgSum( atm_export, a2x_a, n )
   end do
   ! Calculate time-average
   call eshr_EState_TimeAvg( a2x_a, n )

atm_comp_run2:

   if ( sync_clock%curr_time /= my_clock%curr_time+my_clock%TimeStep ) abort
   if ( sync_clock%atm_alarm /= ringing ) abort
   call ESMF_AlarmGet( sync_clock%atm_alarm, next_ring_time=next_time )
   ! Convert ESMF import state to CAM internal input data-type
   call atm_Import_ESMF( x2a_a, cam_in )
   ! Zero out time-average
   call eshr_EState_TimeAvgZero( a2x_a, n )
   do while ( (.not. ESMF_ClockIsStopTime(my_clock)) .and. (my_clock%curr_time .lt. next_time) )
     call cam_run2( cam_out, cam_in )
     call cam_run3( cam_out )
     call cam_run4( cam_out, cam_in, rstwr, nlend )
     call advance_clock( my_clock )
     if ( my_clock%curr_time .lt. next_time )then
       call cam_run1( cam_in, cam_out )
     end do
     ! Convert CAM internal output data-type to ESMF export state
     call atm_Export_ESMF( cam_out, atm_export )
     ! Add to Time-average
     call eshr_EState_TimeAvgSum( atm_export, a2x_a, n )
   end do
   ! Calculate time-average
   call eshr_EState_TimeAvg( a2x_a, n )

Alturnative – run driver at atm/lnd internal time-step

In this case, we use the straightforward CAM component that just runs over a single time step and doesn't do time-averaging. At initialization, the time-step from atm and lnd would have to be sent up from the components, and used for the driver-time-step. Below I also assume that ice and ocean do not do any time-averaging.
Assume:

Driver time-step – 20 min
Ice time-step – 1 hour
atm/land coupling interval – 1 hour

Ocean time-step and coupling interval – 1 day

PROBLEM: The eshr_ calls below need to be inside a gridded component – so they run on the right VM!

 !
 ! Time loop
 !
do while( .not. ESMF_ClockIsStopTime( ESyncClock )  )
   if ( shr_inputinfo_RunModel( initinfo,  atm  ) .and. ESMF_AlarmIsRinging( atm_alarm ) )then
        call eshr_EState_TimeAvg( a2x_a, natm )
        ! Run coupler on atmosphere export state before running...
        call ESMF_CplCompRun ( cc_map_a2l, import=a2x_a, export=a2x_l, rc=rc )
        call ESMF_CplCompRun ( cc_map_a2i, import=a2x_a, export=a2x_i, rc=rc )
        call ESMF_CplCompRun ( cc_map_a2o, import=a2x_a, export=a2x_o, rc=rc )
        call eshr_EState_TimeAvgZero( a2x_a, natm )
   end if
   if ( shr_inputinfo_RunModel( initinfo,  atm  ) )then
      call ESMF_GridCompRun( gc_atm,     import=x2a_a, export=atm_export, ESyncClock, phase=1 )
      call eshr_EState_TimeAvgSum( a2x_a, atm_export, natm )
   end if
   if ( shr_inputinfo_RunModel( initinfo,  lnd  ) .and. ESMF_AlarmIsRinging( lnd_alarm ) )then
       call eshr_EState_TimeAvg( l2x_l, nlnd )
       call ESMF_CplCompRun (  cc_map_l2a, import=l2x_l, export=l2x_a, rc=rc )
        call ESMF_AlarmRingerOff( lnd_alarm )
       call eshr_EState_TimeAvgZero( l2x_l, nlnd )
   end if
   if ( shr_inputinfo_RunModel( initinfo,  lnd  ) )then
     ! Note: cs_l is the composite state pointing to: a2x_l
     call ESMF_GridCompRun ( gc_mrg_x2l, import=cs_l, export=x2l_l, rc=rc ) 
     call ESMF_GridCompRun(	 gc_lnd,     import=x2l_l, export=l2x_l, ESyncClock, phase=1 )
     call ESMF_GridCompRun(	 gc_lnd,	                                      phase=2 )
     call eshr_EState_TimeAvgSum( l2x_l, nlnd )
   end if
   if ( shr_inputinfo_RunModel( initinfo,  ice  ) .and. ESMF_AlarmIsRinging( ice_alarm ) )then
        call ESMF_CplCompRun (  cc_map_i2a, import=i2x_i, export=i2x_a, rc=rc )
        call ESMF_CplCompRun (  cc_map_i2o, import=i2o_i, export=i2o_o, rc=rc )
        ! Note: cs_i is the composite state pointing to: a2x_i, o2i_i
        call ESMF_GridCompRun ( gc_mrg_x2i, import=cs_i, export=x2i_i, rc=rc )
        call ESMF_GridCompRun(  gc_ice,     import=x2i_i, export=i2x_i, ESyncClock, phase=1 )
        call ESMF_GridCompRun(  gc_ice,	                                       phase=2 )
        call ESMF_AlarmRingerOff( ice_alarm )
   end if
   if ( shr_inputinfo_RunModel( initinfo,  ocn  ) .and. ESMF_AlarmIsRinging( ocn_alarm )  )then
        call ESMF_CplCompRun(   cc_map_o2a, import=o2x_o, export=o2x_a, rc=rc)
        call ESMF_CplCompRun (  cc_map_o2i, import=o2x_o, export=o2x_i, rc=rc )
        ! Note: cs_o is the composite state pointing to: i2o_o and a2x_o
        call ESMF_GridCompRun ( gc_mrg_x2o, import=cs_o, export=x2o_o, rc=rc ) 
        call ESMF_GridCompRun(  gc_ocn,     import=x2o_o, export=o2x_o, ESyncClock, phase=1 )
        call ESMF_AlarmRingerOff( ocn_alarm )
   end if
   if ( shr_inputinfo_RunModel( initinfo,  atm  ) .and. ESMF_AlarmIsRinging( atm_alarm ) )then
       ! Note: cs_a is the composite state pointing to: l2x_a, i2x_a, and o2x_a
       call ESMF_GridCompRun( gc_mrg_x2a, import=cs_a, export=x2a_a, rc=rc )
       call ESMF_GridCompRun(	gc_atm,     import=x2a_a,               phase=2 )
       call ESMF_AlarmRingerOff( atm_alarm )
   end if
   ! Advance the clock
   call ESMF_AdvanceClock( ESyncClock )
   ! Write restart information out
   if ( ESMF_AlarmIsRinging( restart ) )then
      call shr_inputinfo_WriteRPointer(  ymd, tod, mpicom, masterproc, initinfo, rest_file )
      call shr_inputinfo_WriteRestart(    rest_file, mpicom, masterproc, initinfo  )
      call shr_timemgr_WriteRestart(  rest_file, mpicom, masterproc, SyncClock )
      call ESMF_AlarmRingerOff( restart )
   end if
end do

Behavior of CAM when running concurrently

Send every radiation time-step (hourly) – mark as previous time-step
Receive on time-step after the send is done
Send and receive if nstep == 0 (mark as current time-step)
Do NOT send or receive if nstep == 1
Code NOT tested when not averaging over radiation time-steps

Sends and receives are done inbetween cam_run1 and cam_run2
Do other models mark data with a different time-stamp than model time-step?
Is this important? For cases we are likely to go to (the hybrid case) we may not need this performance optimization?
Send time maps directly to the atm_alarm. But, receive time implys need another alarm offset by one atmosphere time-step. This makes behavior awkward for the above behavior dependent on alarms.

Put time-stamps on E-States (this could just be integer date and time-of day attributes on States)

 !
 ! Time loop
 !
 do while( .not. ESMF_ClockIsStopTime( ESyncClock )  )
   !
   ! Averaging rules   components are responsible for any averaging done on within 
   ! their coupling interval. Merger will average atmosphere fields for surface components
   ! and put the average on the given surface component import state. Merger will also average
   ! surface data to the atmosphere model coupling interval and put it on the atm import
   !  state
   !
    if ( shr_inputinfo_RunModel( initinfo,  atm  ) .and. ESMF_AlarmIsRinging( atm_alarm ) )then
         ! Now run an atmosphere time-step -- don't use export state until reach next time to run
         call ESMF_GridCompRun( gc_atm,     import=x2a_a, export=a2x_a, ESyncClock, phase=1 )
    end if
    if ( eshr_EState_ReachedTimeStamp( a2x_a, ESyncClock ) )then
         call ESMF_CplCompRun ( cc_map_a2l, import=a2x_a, export=a2x_l, rc=rc )
         call ESMF_CplCompRun ( cc_map_a2i, import=a2x_a, export=a2x_i, rc=rc )
         call ESMF_CplCompRun ( cc_map_a2o, import=a2x_a, export=a2x_o, rc=rc )
    end if
    if ( shr_inputinfo_RunModel( initinfo,  lnd  ) .and. ESMF_AlarmIsRinging( lnd_alarm ) )then
        ! Note: cs_l is the composite state pointing to: a2x_l
        call ESMF_GridCompRun ( gc_mrg_x2l, import=cs_l, export=x2l_l, rc=rc ) 
        call ESMF_GridCompRun(	 gc_lnd,     import=x2l_l, export=l2x_l, ESyncClock, phase=1 )
        call ESMF_GridCompRun(	 gc_lnd,	                                      phase=2 )
        call ESMF_AlarmRingerOff( lnd_alarm )
    end if
    if ( eshr_EState_ReachedTimeStamp( l2x_l, ESyncClock ) )then
       call ESMF_CplCompRun (  cc_map_l2a, import=l2x_l, export=l2x_a, rc=rc )
    end if
    if ( shr_inputinfo_RunModel( initinfo,  ice  ) .and. ESMF_AlarmIsRinging( ice_alarm ) )then
         ! Note: cs_i is the composite state pointing to: a2x_i, o2i_i
         call ESMF_GridCompRun ( gc_mrg_x2i, import=cs_i, export=x2i_i, rc=rc )
         call ESMF_GridCompRun(  gc_ice,     import=x2i_i, export=i2x_i, ESyncClock, phase=1 )
         call ESMF_GridCompRun(  gc_ice,	                                       phase=2 )
         call ESMF_AlarmRingerOff( ice_alarm )
    end if
    if ( eshr_EState_ReachedTimeStamp( i2x_i, ESyncClock ) )then
         call ESMF_CplCompRun (  cc_map_i2a, import=i2x_i, export=i2x_a, rc=rc )
         call ESMF_CplCompRun (  cc_map_i2o, import=i2o_i, export=i2o_o, rc=rc )
    end if
    if ( shr_inputinfo_RunModel( initinfo,  ocn  ) .and. ESMF_AlarmIsRinging( ocn_alarm )  )then
         ! Note: cs_o is the composite state pointing to: i2o_o and a2x_o
         call ESMF_GridCompRun ( gc_mrg_x2o, import=cs_o, export=x2o_o, rc=rc ) 
         call ESMF_GridCompRun(  gc_ocn,     import=x2o_o, export=o2x_o, ESyncClock, phase=1 )
         call ESMF_AlarmRingerOff( ocn_alarm )
    end if
    if ( eshr_EState_ReachedTimeStamp( o2x_o, ESyncClock ) )then
         call ESMF_CplCompRun(   cc_map_o2a, import=o2x_o, export=o2x_a, rc=rc)
         call ESMF_CplCompRun (  cc_map_o2i, import=o2x_o, export=o2x_i, rc=rc )
    end if
    if ( shr_inputinfo_RunModel( initinfo,  atm  ) .and. ESMF_AlarmIsRinging( atm_alarm ) )then
        ! Note: cs_a is the composite state pointing to: l2x_a, i2x_a, and o2x_a
        call ESMF_GridCompRun( gc_mrg_x2a, import=cs_a, export=x2a_a, rc=rc )
        call ESMF_GridCompRun(	gc_atm,     import=x2a_a,               phase=2 )
        call ESMF_AlarmRingerOff( atm_alarm )
    end if
    ! Advance the clock
    call ESMF_AdvanceClock( ESyncClock )
    ! Write restart information out
    if ( ESMF_AlarmIsRinging( restart ) )then
       call shr_inputinfo_WriteRPointer(  ymd, tod, mpicom, masterproc, initinfo, rest_file )
       call shr_inputinfo_WriteRestart(    rest_file, mpicom, masterproc, initinfo  )
       call shr_timemgr_WriteRestart(  rest_file, mpicom, masterproc, SyncClock )
       call ESMF_AlarmRingerOff( restart )
    end if
 end do

Use Ready/Not-Ready flags on E-States

We go back to the case where the driver runs at the atm/lnd internal
time-step. Since, the atmosphere is called every time-step – it can then flag it's import/export states as ready or not.
In this case, we use the straightforward CAM component that just runs over a single time step and doesn't do time-averaging. At initialization, the time-step from atm and lnd would have to be sent up from the components, and used for the driver-time-step. Below I also assume that ice and ocean do not do any time-averaging.
Assume:

Driver time-step – 20 min
Ice time-step – 1 hour
atm/land coupling interval – 1 hour

Ocean time-step and coupling interval – 1 day

PROBLEM: The eshr_ calls below need to be inside a gridded component – so they run on the right VM!

 !
 ! Time loop
 !
do while( .not. ESMF_ClockIsStopTime( ESyncClock )  )
   if ( shr_inputinfo_RunModel( initinfo,  atm  ) .and. eshr_EState_IsReady( a2x_a ) )then
        call eshr_EState_TimeAvg( a2x_a, natm )
        ! Run coupler on atmosphere export state before running...
        call ESMF_CplCompRun ( cc_map_a2l, import=a2x_a, export=a2x_l, rc=rc )
        call ESMF_CplCompRun ( cc_map_a2i, import=a2x_a, export=a2x_i, rc=rc )
        call ESMF_CplCompRun ( cc_map_a2o, import=a2x_a, export=a2x_o, rc=rc )
        call eshr_EState_TimeAvgZero( a2x_a, natm )
   end if
   if ( shr_inputinfo_RunModel( initinfo,  atm  ) )then
      call ESMF_GridCompRun( gc_atm,     import=x2a_a, export=atm_export, ESyncClock, phase=1 )
      call eshr_EState_TimeAvgSum( a2x_a, atm_export, natm )
   end if
   if ( shr_inputinfo_RunModel( initinfo,  lnd  ) .and. eshr_EState_IsReady( l2x_l ) )then
       call eshr_EState_TimeAvg( l2x_l, nlnd )
       call ESMF_CplCompRun (  cc_map_l2a, import=l2x_l, export=l2x_a, rc=rc )
        call ESMF_AlarmRingerOff( lnd_alarm )
       call eshr_EState_TimeAvgZero( l2x_l, nlnd )
   end if
   if ( shr_inputinfo_RunModel( initinfo,  lnd  ) )then
      ! Note: cs_l is the composite state pointing to: a2x_l
      call ESMF_GridCompRun ( gc_mrg_x2l, import=cs_l, export=x2l_l, rc=rc ) 
      call ESMF_GridCompRun(	 gc_lnd,     import=x2l_l, export=l2x_l, ESyncClock, phase=1 )
      call ESMF_GridCompRun(	 gc_lnd,	                                      phase=2 )
      call eshr_EState_TimeAvgSum( l2x_l, nlnd )
   end if
   if ( shr_inputinfo_RunModel( initinfo,  ice  ) .and. eshr_EState_IsReady( i2x_i ) )then
        call ESMF_CplCompRun (  cc_map_i2a, import=i2x_i, export=i2x_a, rc=rc )
        call ESMF_CplCompRun (  cc_map_i2o, import=i2o_i, export=i2o_o, rc=rc )
        ! Note: cs_i is the composite state pointing to: a2x_i, o2i_i
        call ESMF_GridCompRun ( gc_mrg_x2i, import=cs_i, export=x2i_i, rc=rc )
        call ESMF_GridCompRun(  gc_ice,     import=x2i_i, export=i2x_i, ESyncClock, phase=1 )
        call ESMF_GridCompRun(  gc_ice,	                                       phase=2 )
        call ESMF_AlarmRingerOff( ice_alarm )
   end if
   if ( shr_inputinfo_RunModel( initinfo,  ocn  ) .and. eshr_EState_IsReady( o2x_o ) )then
        call ESMF_CplCompRun(   cc_map_o2a, import=o2x_o, export=o2x_a, rc=rc)
        call ESMF_CplCompRun (  cc_map_o2i, import=o2x_o, export=o2x_i, rc=rc )
        ! Note: cs_o is the composite state pointing to: i2o_o and a2x_o
        call ESMF_GridCompRun ( gc_mrg_x2o, import=cs_o, export=x2o_o, rc=rc ) 
        call ESMF_GridCompRun(  gc_ocn,     import=x2o_o, export=o2x_o, ESyncClock, phase=1 )
        call ESMF_AlarmRingerOff( ocn_alarm )
   end if
   if ( shr_inputinfo_RunModel( initinfo,  atm  ) )then
       ! Note: cs_a is the composite state pointing to: l2x_a, i2x_a, and o2x_a
       call ESMF_GridCompRun( gc_mrg_x2a, import=cs_a, export=x2a_a, rc=rc )
       call ESMF_GridCompRun(	gc_atm,     import=x2a_a,               phase=2 )
   end if
   ! Advance the clock
   call ESMF_AdvanceClock( ESyncClock )
   ! Write restart information out
   if ( ESMF_AlarmIsRinging( restart ) )then
      call shr_inputinfo_WriteRPointer(  ymd, tod, mpicom, masterproc, initinfo, rest_file )
      call shr_inputinfo_WriteRestart(    rest_file, mpicom, masterproc, initinfo  )
      call shr_timemgr_WriteRestart(  rest_file, mpicom, masterproc, SyncClock )
      call ESMF_AlarmRingerOff( restart )
   end if
end do