Update OMP (CICE-Consortium#680)

* Add narwhal intel, gnu, cray, aocc
Add perf_suite.ts

* update narwhal_cray and perf_suite

* Review and update OMP implementation

- Fix call to timers in block loops
- Fix some OMP Private variables
- Test OMP Scheduling, add SCHEDULE(runtime) to some OMP loops
- Review column and advection OMP implementation
- ADD OMP_TIMERS CPP option (temporary) to time threaded sections
- Add timer_tmp timers (temporary)
- Add omp_suite.ts test suite
- Add ability to set OMP_SCHEDULE via options (ompscheds, ompscheds1, ompschedd1)

* - Review diagnostics OMP implementation
- Add timer_stats namelist to turn on extra timer output information
- Add ICE_BFBTYPE and update bit-for-bit comparison logic in scripts
- Update qc and logbfb testing
- Remove logbfb and qchkf tests, add cmplog, cmplogrest, cmprest set_env files to set ICE_BFBTYPE
- Update documentation

* Update EVP OMP implementation

* - Refactor puny/pi scalars in eap dynamics to improve performance
- Update OMP in evp and eap

* Clean up

* Comment out temporary timers

* Update OMP env variables on Narwhal

* Update gaffney OMP_STACKSIZE

* update OMP_STACKSIZE on cori

* Update Onyx OMP_STACKSIZE
Update documentation

* Update OMP_STACKSIZE on mustang

* - Update Tsfc values on land in various places in the code, was affecting testing.  Specifically fix upwind advection.
- Comment out OMP in ice_dyn_evp_1d_kernel, was producing non bit-for-bit results with different thread counts

cicecore/cicedynB/analysis/ice_diagnostics.F90

@@ -720,7 +720,7 @@ subroutine runtime_diags (dt)
             fisoon(n) = fisoon(n)*dt
             fisoos(n) = fisoos(n)*dt
 
-            !$OMP PARALLEL DO PRIVATE(iblk,i,j)
+            !$OMP PARALLEL DO PRIVATE(iblk,i,j,k)
             do iblk = 1, nblocks
                do j = 1, ny_block
                do i = 1, nx_block
@@ -1214,7 +1214,7 @@ subroutine init_mass_diags
 
       if (tr_iso) then
          do n=1,n_iso
-            !$OMP PARALLEL DO PRIVATE(iblk,i,j)
+            !$OMP PARALLEL DO PRIVATE(iblk,i,j,k)
             do iblk = 1, nblocks
                do j = 1, ny_block
                do i = 1, nx_block
@@ -1317,7 +1317,6 @@ subroutine total_energy (work)
       if (icepack_warnings_aborted()) call abort_ice(error_message=subname, &
          file=__FILE__, line=__LINE__)
 
-! MHRI: CHECK THIS OMP
       !$OMP PARALLEL DO PRIVATE(iblk,i,j,n,k,ij,icells,indxi,indxj)
       do iblk = 1, nblocks
 
@@ -1405,7 +1404,6 @@ subroutine total_salt (work)
       if (icepack_warnings_aborted()) call abort_ice(error_message=subname, &
          file=__FILE__, line=__LINE__)
 
-! MHRI: CHECK THIS OMP
       !$OMP PARALLEL DO PRIVATE(iblk,i,j,n,k,ij,icells,indxi,indxj)
       do iblk = 1, nblocks
 
@@ -1513,7 +1511,8 @@ subroutine init_diags
             if (abs(latpnt(n)) < c360 .and. abs(lonpnt(n)) < c360) then
 
             ! MDT, 09/2017: Comment out OpenMP directives since loop is not thread-safe
-            !!$OMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,latdis,londis,totdis)
+            ! This is computing closest point, Could add a CRITICAL but it's just initialization
+            !!$XXXOMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,latdis,londis,totdis)
             do iblk = 1, nblocks
                this_block = get_block(blocks_ice(iblk),iblk)         
                ilo = this_block%ilo
@@ -1538,7 +1537,7 @@ subroutine init_diags
                enddo            ! i
                enddo            ! j
             enddo               ! iblk
-            !!$OMP END PARALLEL DO
+            !!$XXXOMP END PARALLEL DO
 
             endif
 

cicecore/cicedynB/analysis/ice_history.F90

@@ -1889,7 +1889,6 @@ subroutine accum_hist (dt)
       ! increment field
       !---------------------------------------------------------------
 
-! MHRI: CHECK THIS OMP ... Maybe ok after "dfresh,dfsalt" added
       !$OMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block, &
       !$OMP             k,n,qn,ns,sn,rho_ocn,rho_ice,Tice,Sbr,phi,rhob,dfresh,dfsalt, &
       !$OMP             worka,workb,worka3,Tinz4d,Sinz4d,Tsnz4d)