From 8295378637876a4f3142ca9758d8fdad0e7d446a Mon Sep 17 00:00:00 2001 From: prkkumar Date: Wed, 31 May 2023 19:50:00 -0700 Subject: [PATCH 01/22] relative error based steady state --- Source/main.cpp | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/Source/main.cpp b/Source/main.cpp index a9e6740..1da6461 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -504,11 +504,29 @@ void main_main (c_FerroX& rFerroX) } // Check if steady state has reached - MultiFab::Copy(Phidiff, PoissonPhi, 0, 0, 1, 0); - MultiFab::Subtract(Phidiff, PoissonPhi_Old, 0, 0, 1, 0); - Real phi_err = Phidiff.norm0(); + //MultiFab::Copy(Phidiff, PoissonPhi, 0, 0, 1, 0); + //MultiFab::Subtract(Phidiff, PoissonPhi_Old, 0, 0, 1, 0); - if (phi_err < phi_tolerance) { + Real phi_max = PoissonPhi_Old.norm0(); + + for (MFIter mfi(PoissonPhi); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.growntilebox(1); + + const Array4& Phi = PoissonPhi.array(mfi); + const Array4& PhiOld = PoissonPhi_Old.array(mfi); + const Array4& Phi_err = Phidiff.array(mfi); + + + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) + { + Phi_err(i,j,k) = amrex::Math::abs(Phi(i,j,k) - PhiOld(i,j,k)) / phi_max; + }); + } + + Real max_phi_err = Phidiff.norm0(); + + if (max_phi_err < phi_tolerance) { steady_state_step = step; inc_step = step; } @@ -516,7 +534,7 @@ void main_main (c_FerroX& rFerroX) //Copy PoissonPhi to PoissonPhi_Old to calculate difference at the next iteration MultiFab::Copy(PoissonPhi_Old, PoissonPhi, 0, 0, 1, 0); - amrex::Print() << "Steady state check : (phi(t) - phi(t-1)).norm0() = " << phi_err << std::endl; + amrex::Print() << "Steady state check : (phi(t) - phi(t-1)).norm0() = " << max_phi_err << std::endl; // Calculate E from Phi From 5490ead78ebb443cce0236e3bf1fcc9028e7634b Mon Sep 17 00:00:00 2001 From: prkkumar Date: Thu, 29 Jun 2023 15:57:25 -0700 Subject: [PATCH 02/22] Nodal Poisson --- Source/Solver/DerivativeAlgorithm.H | 382 +++++++++++++++------------- Source/main.cpp | 246 +++++++++++------- 2 files changed, 359 insertions(+), 269 deletions(-) diff --git a/Source/Solver/DerivativeAlgorithm.H b/Source/Solver/DerivativeAlgorithm.H index 8f3256d..ccc2e76 100644 --- a/Source/Solver/DerivativeAlgorithm.H +++ b/Source/Solver/DerivativeAlgorithm.H @@ -12,13 +12,18 @@ using namespace FerroX; amrex::GpuArrayconst& prob_lo, amrex::GpuArrayconst& prob_hi) { - if (z_lo < prob_lo[2]){ // bottom metal - return (-4.*F(i,j,k-1) + 3.*F(i,j,k) + F(i,j,k+1)) / (3. * dx[2]); - } else if (z_hi > prob_hi[2]){ // top metal - return (4.*F(i,j,k+1) - 3.*F(i,j,k) - F(i,j,k-1)) / (3. * dx[2]); - } else { // inside stack - return (F(i,j,k+1) - F(i,j,k-1)) / (2. * dx[2]); - } + //if (z_lo < prob_lo[2]){ // bottom metal + // return (-4.*F(i,j,k-1) + 3.*F(i,j,k) + F(i,j,k+1)) / (3. * dx[2]); + //} else if (z_hi > prob_hi[2]){ // top metal + // return (4.*F(i,j,k+1) - 3.*F(i,j,k) - F(i,j,k-1)) / (3. * dx[2]); + //} else { // inside stack + // return (F(i,j,k+1) - F(i,j,k-1)) / (2. * dx[2]); + //} + return ( F(i,j, k+1) - F(i,j,k) + + F(i+1,j, k+1) - F(i+1,j,k) + + F(i,j+1, k+1) - F(i,j+1,k) + + F(i+1,j+1,k+1) - F(i+1,j+1,k) + )/4./dx[2]; } /** @@ -27,7 +32,11 @@ using namespace FerroX; static amrex::Real DFDx ( amrex::Array4 const& F, int const i, int const j, int const k, amrex::GpuArray dx) { - return (F(i+1,j,k) - F(i-1,j,k))/(2.*dx[0]); + return ( F(i+1,j, k) - F(i,j,k) + + F(i+1,j+1, k) - F(i,j+1,k) + + F(i+1,j, k+1) - F(i,j,k+1) + + F(i+1,j+1,k+1) - F(i,j+1,k+1) + )/4./dx[0]; } /** @@ -36,7 +45,11 @@ using namespace FerroX; static amrex::Real DFDy ( amrex::Array4 const& F, int const i, int const j, int const k, amrex::GpuArray dx) { - return (F(i,j+1,k) - F(i,j-1,k))/(2.*dx[1]); + return ( F(i, j+1, k) - F(i,j,k) + + F(i+1,j+1, k) - F(i+1,j,k) + + F(i ,j+1,k+1) - F(i,j,k+1) + + F(i+1,j+1,k+1) - F(i+1,j,k+1) + )/4./dx[1]; } /** @@ -47,62 +60,67 @@ using namespace FerroX; amrex::Array4 const& mask, int const i, int const j, int const k, amrex::GpuArray dx ) { - if (mask(i-1,j,k) != 0.0 && mask(i,j,k) == 0.0) { //FE lower boundary - - if(P_BC_flag_lo[0] == 0){ - Real F_lo = 0.0; - return (-4.*F_lo + 3.*F(i,j,k) + F(i+1,j,k))/(3.*dx[0]);//2nd order using three point stencil using 0, pOld(i,j,k), and pOld(i+1,j,k) - - } else if (P_BC_flag_lo[0] == 1){ - - Real F_lo = F(i,j,k)/(1 + dx[0]/2/lambda); - return (dx[0]*F_lo/lambda - F(i,j,k) + F(i+1,j,k))/(2.*dx[0]); // dP/dz = P_lo/lambda; - - // Real F_lo = (9. * F(i,j,k) - F(i,j,k+1)) / (3. * dx[2] / lambda + 8.); // derived with 2nd order one-sided 1st derivative - // return -(dx[2]*F_lo/lambda - F(i,j,k) + F(i,j,k+1))/(2.*dx[2]);// dP/dz = P_lo/lambda; - - } else if (P_BC_flag_lo[0] == 2){ - return ( - F(i,j,k) + F(i+1,j,k))/(2.*dx[0]); //dPdx = 0. - - } else if (P_BC_flag_lo[0] == 3){ - return (- F(i-1,j,k) + F(i+1,j,k))/(2.*dx[0]); //periodic - - } else { - amrex::Abort("Wrong flag of the lower x polarization boundary condition!!"); - return 0.0; - } - - } else if (mask(i+1,j,k) != 0.0 && mask(i,j,k) == 0.0){ // FE higher boundary - - if(P_BC_flag_hi[0] == 0){ - Real F_hi = 0.0; - return (4.*F_hi - 3.*F(i,j,k) - F(i-1,j,k))/(3.*dx[0]);//2nd order using three point stencil using 0, pOld(i,j,k), and pOld(i-1,j,k) - - } else if (P_BC_flag_hi[0] == 1){ - - Real F_hi = F(i,j,k)/(1 - dx[0]/2/lambda); - return (dx[0]*F_hi/lambda + F(i,j,k) - F(i-1,j,k))/(2.*dx[0]);//dPdz = P_hi/lambda; - - // Real F_hi = (9. * F(i,j,k) - F(i,j,k-1)) / ( - 3. * dx[2] / lambda + 8.); // derived with 2nd order one-sided 1st derivative - // return -(dx[2]*F_hi/lambda + F(i,j,k) - F(i,j,k-1))/(2.*dx[2]);//dPdz = P_hi/lambda; - - } else if (P_BC_flag_hi[0] == 2){ - return (F(i,j,k) - F(i-1,j,k))/(2.*dx[0]); //dPdx = 0. - - } else if (P_BC_flag_hi[0] == 3){ - return (F(i+1,j,k) - F(i-1,j,k))/(2.*dx[0]); //Periodic - - } else { - amrex::Abort("Wrong flag of the higher x polarization boundary condition!!"); - return 0.0; - } - - } else if (mask(i,j,k) == 0.0) { // inside FE - return (F(i+1,j,k) - F(i-1,j,k))/(2.*dx[0]); - - } else { - return 0.0; - } + // if (mask(i-1,j,k) != 0.0 && mask(i,j,k) == 0.0) { //FE lower boundary + // + // if(P_BC_flag_lo[0] == 0){ + // Real F_lo = 0.0; + // return (-4.*F_lo + 3.*F(i,j,k) + F(i+1,j,k))/(3.*dx[0]);//2nd order using three point stencil using 0, pOld(i,j,k), and pOld(i+1,j,k) + + // } else if (P_BC_flag_lo[0] == 1){ + + // Real F_lo = F(i,j,k)/(1 + dx[0]/2/lambda); + // return (dx[0]*F_lo/lambda - F(i,j,k) + F(i+1,j,k))/(2.*dx[0]); // dP/dz = P_lo/lambda; + + // // Real F_lo = (9. * F(i,j,k) - F(i,j,k+1)) / (3. * dx[2] / lambda + 8.); // derived with 2nd order one-sided 1st derivative + // // return -(dx[2]*F_lo/lambda - F(i,j,k) + F(i,j,k+1))/(2.*dx[2]);// dP/dz = P_lo/lambda; + + // } else if (P_BC_flag_lo[0] == 2){ + // return ( - F(i,j,k) + F(i+1,j,k))/(2.*dx[0]); //dPdx = 0. + + // } else if (P_BC_flag_lo[0] == 3){ + // return (- F(i-1,j,k) + F(i+1,j,k))/(2.*dx[0]); //periodic + + // } else { + // amrex::Abort("Wrong flag of the lower x polarization boundary condition!!"); + // return 0.0; + // } + + // } else if (mask(i+1,j,k) != 0.0 && mask(i,j,k) == 0.0){ // FE higher boundary + + // if(P_BC_flag_hi[0] == 0){ + // Real F_hi = 0.0; + // return (4.*F_hi - 3.*F(i,j,k) - F(i-1,j,k))/(3.*dx[0]);//2nd order using three point stencil using 0, pOld(i,j,k), and pOld(i-1,j,k) + + // } else if (P_BC_flag_hi[0] == 1){ + // + // Real F_hi = F(i,j,k)/(1 - dx[0]/2/lambda); + // return (dx[0]*F_hi/lambda + F(i,j,k) - F(i-1,j,k))/(2.*dx[0]);//dPdz = P_hi/lambda; + + // // Real F_hi = (9. * F(i,j,k) - F(i,j,k-1)) / ( - 3. * dx[2] / lambda + 8.); // derived with 2nd order one-sided 1st derivative + // // return -(dx[2]*F_hi/lambda + F(i,j,k) - F(i,j,k-1))/(2.*dx[2]);//dPdz = P_hi/lambda; + + // } else if (P_BC_flag_hi[0] == 2){ + // return (F(i,j,k) - F(i-1,j,k))/(2.*dx[0]); //dPdx = 0. + + // } else if (P_BC_flag_hi[0] == 3){ + // return (F(i+1,j,k) - F(i-1,j,k))/(2.*dx[0]); //Periodic + + // } else { + // amrex::Abort("Wrong flag of the higher x polarization boundary condition!!"); + // return 0.0; + // } + // + // } else if (mask(i,j,k) == 0.0) { // inside FE + // return (F(i+1,j,k) - F(i-1,j,k))/(2.*dx[0]); + + // } else { + // return 0.0; + // } + return ( F(i,j-1,k-1) - F(i-1,j-1,k-1) + + F(i,j, k-1) - F(i-1,j ,k-1) + + F(i,j-1, k) - F(i-1,j-1,k) + + F(i,j, k) - F(i-1,j, k) + )/4./dx[0]; } /** @@ -114,62 +132,67 @@ using namespace FerroX; int const i, int const j, int const k, amrex::GpuArray dx ) { - if (mask(i,j-1,k) != 0.0 && mask(i,j,k) == 0.0) { //FE lower boundary - - if(P_BC_flag_lo[1] == 0){ - Real F_lo = 0.0; - return (-4.*F_lo + 3.*F(i,j,k) + F(i,j+1,k))/(3.*dx[1]);//2nd order using three point stencil using 0, pOld(i,j,k), and pOld(i,j,k+1) - - } else if (P_BC_flag_lo[1] == 1){ - - Real F_lo = F(i,j,k)/(1 + dx[1]/2/lambda); - return (dx[1]*F_lo/lambda - F(i,j,k) + F(i,j+1,k))/(2.*dx[1]); // dP/dz = P_lo/lambda; - - // Real F_lo = (9. * F(i,j,k) - F(i,j,k+1)) / (3. * dx[2] / lambda + 8.); // derived with 2nd order one-sided 1st derivative - // return -(dx[2]*F_lo/lambda - F(i,j,k) + F(i,j,k+1))/(2.*dx[2]);// dP/dz = P_lo/lambda; - - } else if (P_BC_flag_lo[1] == 2){ - return ( - F(i,j,k) + F(i,j+1,k))/(2.*dx[1]); //dPdy = 0. - - } else if (P_BC_flag_lo[1] == 3){ - return ( - F(i,j-1,k) + F(i,j+1,k))/(2.*dx[1]); //Periodic - - } else { - amrex::Abort("Wrong flag of the lower polarization boundary condition!!"); - return 0.0; - } - - } else if (mask(i,j+1,k) != 0.0 && mask(i,j,k) == 0.0){ // FE higher boundary - - if(P_BC_flag_hi[1] == 0){ - Real F_hi = 0.0; - return (4.*F_hi - 3.*F(i,j,k) - F(i,j-1,k))/(3.*dx[1]);//2nd order using three point stencil using 0, pOld(i,j,k), and pOld(i,j,k-1) - - } else if (P_BC_flag_hi[1] == 1){ - - Real F_hi = F(i,j,k)/(1 - dx[1]/2/lambda); - return (dx[1]*F_hi/lambda + F(i,j,k) - F(i,j-1,k))/(2.*dx[1]);//dPdz = P_hi/lambda; - - // Real F_hi = (9. * F(i,j,k) - F(i,j,k-1)) / ( - 3. * dx[2] / lambda + 8.); // derived with 2nd order one-sided 1st derivative - // return -(dx[2]*F_hi/lambda + F(i,j,k) - F(i,j,k-1))/(2.*dx[2]);//dPdz = P_hi/lambda; - - } else if (P_BC_flag_hi[1] == 2){ - return (F(i,j,k) - F(i,j-1,k))/(2.*dx[1]); //dPdy = 0. - - } else if (P_BC_flag_hi[1] == 3){ - return (F(i,j+1,k) - F(i,j-1,k))/(2.*dx[1]); //Periodic - - } else { - amrex::Abort("Wrong flag of the higher y polarization boundary condition!!"); - return 0.0; - } - - } else if (mask(i,j,k) == 0.0) { // inside FE - return (F(i,j+1,k) - F(i,j-1,k))/(2.*dx[1]); - - } else { - return 0.0; - } +// if (mask(i,j-1,k) != 0.0 && mask(i,j,k) == 0.0) { //FE lower boundary +// +// if(P_BC_flag_lo[1] == 0){ +// Real F_lo = 0.0; +// return (-4.*F_lo + 3.*F(i,j,k) + F(i,j+1,k))/(3.*dx[1]);//2nd order using three point stencil using 0, pOld(i,j,k), and pOld(i,j,k+1) +// +// } else if (P_BC_flag_lo[1] == 1){ +// +// Real F_lo = F(i,j,k)/(1 + dx[1]/2/lambda); +// return (dx[1]*F_lo/lambda - F(i,j,k) + F(i,j+1,k))/(2.*dx[1]); // dP/dz = P_lo/lambda; +// +// // Real F_lo = (9. * F(i,j,k) - F(i,j,k+1)) / (3. * dx[2] / lambda + 8.); // derived with 2nd order one-sided 1st derivative +// // return -(dx[2]*F_lo/lambda - F(i,j,k) + F(i,j,k+1))/(2.*dx[2]);// dP/dz = P_lo/lambda; +// +// } else if (P_BC_flag_lo[1] == 2){ +// return ( - F(i,j,k) + F(i,j+1,k))/(2.*dx[1]); //dPdy = 0. +// +// } else if (P_BC_flag_lo[1] == 3){ +// return ( - F(i,j-1,k) + F(i,j+1,k))/(2.*dx[1]); //Periodic +// +// } else { +// amrex::Abort("Wrong flag of the lower polarization boundary condition!!"); +// return 0.0; +// } +// +// } else if (mask(i,j+1,k) != 0.0 && mask(i,j,k) == 0.0){ // FE higher boundary +// +// if(P_BC_flag_hi[1] == 0){ +// Real F_hi = 0.0; +// return (4.*F_hi - 3.*F(i,j,k) - F(i,j-1,k))/(3.*dx[1]);//2nd order using three point stencil using 0, pOld(i,j,k), and pOld(i,j,k-1) +// +// } else if (P_BC_flag_hi[1] == 1){ +// +// Real F_hi = F(i,j,k)/(1 - dx[1]/2/lambda); +// return (dx[1]*F_hi/lambda + F(i,j,k) - F(i,j-1,k))/(2.*dx[1]);//dPdz = P_hi/lambda; +// +// // Real F_hi = (9. * F(i,j,k) - F(i,j,k-1)) / ( - 3. * dx[2] / lambda + 8.); // derived with 2nd order one-sided 1st derivative +// // return -(dx[2]*F_hi/lambda + F(i,j,k) - F(i,j,k-1))/(2.*dx[2]);//dPdz = P_hi/lambda; +// +// } else if (P_BC_flag_hi[1] == 2){ +// return (F(i,j,k) - F(i,j-1,k))/(2.*dx[1]); //dPdy = 0. +// +// } else if (P_BC_flag_hi[1] == 3){ +// return (F(i,j+1,k) - F(i,j-1,k))/(2.*dx[1]); //Periodic +// +// } else { +// amrex::Abort("Wrong flag of the higher y polarization boundary condition!!"); +// return 0.0; +// } +// +// } else if (mask(i,j,k) == 0.0) { // inside FE +// return (F(i,j+1,k) - F(i,j-1,k))/(2.*dx[1]); +// +// } else { +// return 0.0; +// } + return ( F(i-1,j,k-1) - F(i-1,j-1,k-1) + + F(i-1,j, k) - F(i-1,j-1,k) + + F(i, j,k-1) - F(i,j-1,k-1) + + F(i, j, k) - F(i,j-1, k) + )/4./dx[1]; } /** @@ -181,62 +204,67 @@ using namespace FerroX; int const i, int const j, int const k, amrex::GpuArray dx ) { - if (mask(i,j,k-1) != 0.0 && mask(i,j,k) == 0.0) { //FE lower boundary - - if(P_BC_flag_lo[2] == 0){ - Real F_lo = 0.0; - return (-4.*F_lo + 3.*F(i,j,k) + F(i,j,k+1))/(3.*dx[2]);//2nd order using three point stencil using 0, pOld(i,j,k), and pOld(i,j,k+1) - - } else if (P_BC_flag_lo[2] == 1){ - - Real F_lo = F(i,j,k)/(1 + dx[2]/2/lambda); - return (dx[2]*F_lo/lambda - F(i,j,k) + F(i,j,k+1))/(2.*dx[2]); // dP/dz = P_lo/lambda; - - // Real F_lo = (9. * F(i,j,k) - F(i,j,k+1)) / (3. * dx[2] / lambda + 8.); // derived with 2nd order one-sided 1st derivative - // return -(dx[2]*F_lo/lambda - F(i,j,k) + F(i,j,k+1))/(2.*dx[2]);// dP/dz = P_lo/lambda; - - } else if (P_BC_flag_lo[2] == 2){ - return ( - F(i,j,k) + F(i,j,k+1))/(2.*dx[2]); //dPdz = 0. - - } else if (P_BC_flag_lo[2] == 3){ - return ( - F(i,j,k-1) + F(i,j,k+1))/(2.*dx[2]); //No BC - - } else { - amrex::Abort("Wrong flag of the lower polarization boundary condition!!"); - return 0.0; - } - - } else if ( mask(i,j,k+1) != 0.0 && mask(i,j,k) == 0.0 ){ // FE higher boundary - - if(P_BC_flag_hi[2] == 0){ - Real F_hi = 0.0; - return (4.*F_hi - 3.*F(i,j,k) - F(i,j,k-1))/(3.*dx[2]);//2nd order using three point stencil using 0, pOld(i,j,k), and pOld(i,j,k-1) - - } else if (P_BC_flag_hi[2] == 1){ - - Real F_hi = F(i,j,k)/(1 - dx[2]/2/lambda); - return (dx[2]*F_hi/lambda + F(i,j,k) - F(i,j,k-1))/(2.*dx[2]);//dPdz = P_hi/lambda; - - // Real F_hi = (9. * F(i,j,k) - F(i,j,k-1)) / ( - 3. * dx[2] / lambda + 8.); // derived with 2nd order one-sided 1st derivative - // return -(dx[2]*F_hi/lambda + F(i,j,k) - F(i,j,k-1))/(2.*dx[2]);//dPdz = P_hi/lambda; - - } else if (P_BC_flag_hi[2] == 2){ - return (F(i,j,k) - F(i,j,k-1))/(2.*dx[2]); //dPdz = 0. - - } else if (P_BC_flag_hi[2] == 3){ - return (0. - F(i,j,k-1))/(2.*dx[2]); //No BC - - } else { - amrex::Abort("Wrong flag of the higher polarization boundary condition!!"); - return 0.0; - } - - } else if (mask(i,j,k) == 0.0) { // inside FE - return (F(i,j,k+1) - F(i,j,k-1))/(2.*dx[2]); - - } else { - return 0.0; - } +// if (mask(i,j,k-1) != 0.0 && mask(i,j,k) == 0.0) { //FE lower boundary +// +// if(P_BC_flag_lo[2] == 0){ +// Real F_lo = 0.0; +// return (-4.*F_lo + 3.*F(i,j,k) + F(i,j,k+1))/(3.*dx[2]);//2nd order using three point stencil using 0, pOld(i,j,k), and pOld(i,j,k+1) +// +// } else if (P_BC_flag_lo[2] == 1){ +// +// Real F_lo = F(i,j,k)/(1 + dx[2]/2/lambda); +// return (dx[2]*F_lo/lambda - F(i,j,k) + F(i,j,k+1))/(2.*dx[2]); // dP/dz = P_lo/lambda; +// +// // Real F_lo = (9. * F(i,j,k) - F(i,j,k+1)) / (3. * dx[2] / lambda + 8.); // derived with 2nd order one-sided 1st derivative +// // return -(dx[2]*F_lo/lambda - F(i,j,k) + F(i,j,k+1))/(2.*dx[2]);// dP/dz = P_lo/lambda; +// +// } else if (P_BC_flag_lo[2] == 2){ +// return ( - F(i,j,k) + F(i,j,k+1))/(2.*dx[2]); //dPdz = 0. +// +// } else if (P_BC_flag_lo[2] == 3){ +// return ( - F(i,j,k-1) + F(i,j,k+1))/(2.*dx[2]); //No BC +// +// } else { +// amrex::Abort("Wrong flag of the lower polarization boundary condition!!"); +// return 0.0; +// } +// +// } else if ( mask(i,j,k+1) != 0.0 && mask(i,j,k) == 0.0 ){ // FE higher boundary +// +// if(P_BC_flag_hi[2] == 0){ +// Real F_hi = 0.0; +// return (4.*F_hi - 3.*F(i,j,k) - F(i,j,k-1))/(3.*dx[2]);//2nd order using three point stencil using 0, pOld(i,j,k), and pOld(i,j,k-1) +// +// } else if (P_BC_flag_hi[2] == 1){ +// +// Real F_hi = F(i,j,k)/(1 - dx[2]/2/lambda); +// return (dx[2]*F_hi/lambda + F(i,j,k) - F(i,j,k-1))/(2.*dx[2]);//dPdz = P_hi/lambda; +// +// // Real F_hi = (9. * F(i,j,k) - F(i,j,k-1)) / ( - 3. * dx[2] / lambda + 8.); // derived with 2nd order one-sided 1st derivative +// // return -(dx[2]*F_hi/lambda + F(i,j,k) - F(i,j,k-1))/(2.*dx[2]);//dPdz = P_hi/lambda; +// +// } else if (P_BC_flag_hi[2] == 2){ +// return (F(i,j,k) - F(i,j,k-1))/(2.*dx[2]); //dPdz = 0. +// +// } else if (P_BC_flag_hi[2] == 3){ +// return (0. - F(i,j,k-1))/(2.*dx[2]); //No BC +// +// } else { +// amrex::Abort("Wrong flag of the higher polarization boundary condition!!"); +// return 0.0; +// } +// +// } else if (mask(i,j,k) == 0.0) { // inside FE +// return (F(i,j,k+1) - F(i,j,k-1))/(2.*dx[2]); +// +// } else { +// return 0.0; +// } + return ( F(i-1,j-1,k) - F(i-1,j-1,k-1) + + F(i-1,j, k) - F(i-1,j, k-1) + + F(i, j-1,k) - F(i, j-1,k-1) + + F(i, j, k) - F(i, j, k-1) + )/4./dx[2]; } // ///** diff --git a/Source/main.cpp b/Source/main.cpp index 1da6461..09fed8a 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #ifdef AMREX_USE_EB #include #endif @@ -47,6 +48,7 @@ void main_main (c_FerroX& rFerroX) auto& rGprop = rFerroX.get_GeometryProperties(); auto& geom = rGprop.geom; auto& ba = rGprop.ba; + const BoxArray& nba = amrex::convert(ba,IntVect::TheNodeVector()); auto& dm = rGprop.dm; auto& is_periodic = rGprop.is_periodic; auto& prob_lo = rGprop.prob_lo; @@ -112,6 +114,15 @@ void main_main (c_FerroX& rFerroX) MultiFab PoissonPhi_Prev(ba, dm, 1, 1); MultiFab PhiErr(ba, dm, 1, 1); MultiFab Phidiff(ba, dm, 1, 1); + + //For nodal Poisson + MultiFab Nodal_PoissonRHS(nba, dm, 1, 0); + MultiFab Nodal_PoissonPhi(nba, dm, 1, 1); + MultiFab Nodal_PoissonPhi_Old(nba, dm, 1, 1); + MultiFab Nodal_PoissonPhi_Prev(nba, dm, 1, 1); + MultiFab Nodal_PhiErr(nba, dm, 1, 1); + MultiFab Nodal_Phidiff(nba, dm, 1, 1); + MultiFab Ex(ba, dm, 1, 0); MultiFab Ey(ba, dm, 1, 0); MultiFab Ez(ba, dm, 1, 0); @@ -119,6 +130,12 @@ void main_main (c_FerroX& rFerroX) MultiFab hole_den(ba, dm, 1, 0); MultiFab e_den(ba, dm, 1, 0); MultiFab charge_den(ba, dm, 1, 0); + + //Nodal rho + MultiFab Nodal_hole_den(nba, dm, 1, 0); + MultiFab Nodal_e_den(nba, dm, 1, 0); + MultiFab Nodal_charge_den(nba, dm, 1, 0); + MultiFab MaterialMask(ba, dm, 1, 1); MultiFab tphaseMask(ba, dm, 1, 1); MultiFab angle_alpha(ba, dm, 1, 0); @@ -220,44 +237,77 @@ void main_main (c_FerroX& rFerroX) pMLMG->setVerbose(mlmg_verbosity); #else - std::unique_ptr p_mlabec; - p_mlabec = std::make_unique(); - p_mlabec->define({geom}, {ba}, {dm}, info); +// std::unique_ptr p_mlabec; +// p_mlabec = std::make_unique(); +// p_mlabec->define({geom}, {ba}, {dm}, info); +// +// //Force singular system to be solvable +// p_mlabec->setEnforceSingularSolvable(false); +// +// p_mlabec->setMaxOrder(linop_maxorder); +// +// p_mlabec->setDomainBC(LinOpBCType_2d[0], LinOpBCType_2d[1]); +// +// if(some_constant_inhomogeneous_boundaries) +// { +// Fill_Constant_Inhomogeneous_Boundaries(rFerroX, PoissonPhi); +// } +// if(some_functionbased_inhomogeneous_boundaries) +// { +// Fill_FunctionBased_Inhomogeneous_Boundaries(rFerroX, PoissonPhi, time); +// } +// PoissonPhi.FillBoundary(geom.periodicity()); +// +// // set Dirichlet BC by reading in the ghost cell values +// p_mlabec->setLevelBC(amrlev, &PoissonPhi); +// +// // (A*alpha_cc - B * div beta grad) phi = rhs +// p_mlabec->setScalars(-1.0, 1.0); // A = -1.0, B = 1.0; solving (-alpha - div beta grad) phi = RHS +// p_mlabec->setBCoeffs(amrlev, amrex::GetArrOfConstPtrs(beta_face)); +// +// //Declare MLMG object +// pMLMG = std::make_unique(*p_mlabec); +// pMLMG->setVerbose(mlmg_verbosity); + + //Nodal Poisson + std::unique_ptr p_mlnode; + p_mlnode = std::make_unique(); + p_mlnode->define({geom}, {ba}, {dm}, info); //Force singular system to be solvable - p_mlabec->setEnforceSingularSolvable(false); + p_mlnode->setEnforceSingularSolvable(false); - p_mlabec->setMaxOrder(linop_maxorder); + p_mlnode->setMaxOrder(linop_maxorder); - p_mlabec->setDomainBC(LinOpBCType_2d[0], LinOpBCType_2d[1]); + p_mlnode->setDomainBC(LinOpBCType_2d[0], LinOpBCType_2d[1]); if(some_constant_inhomogeneous_boundaries) { - Fill_Constant_Inhomogeneous_Boundaries(rFerroX, PoissonPhi); + Fill_Constant_Inhomogeneous_Boundaries(rFerroX, Nodal_PoissonPhi); } if(some_functionbased_inhomogeneous_boundaries) { - Fill_FunctionBased_Inhomogeneous_Boundaries(rFerroX, PoissonPhi, time); + Fill_FunctionBased_Inhomogeneous_Boundaries(rFerroX, Nodal_PoissonPhi, time); } - PoissonPhi.FillBoundary(geom.periodicity()); + Nodal_PoissonPhi.FillBoundary(geom.periodicity()); // set Dirichlet BC by reading in the ghost cell values - p_mlabec->setLevelBC(amrlev, &PoissonPhi); + //p_mlnode->setLevelBC(amrlev, &Nodal_PoissonPhi); - // (A*alpha_cc - B * div beta grad) phi = rhs - p_mlabec->setScalars(-1.0, 1.0); // A = -1.0, B = 1.0; solving (-alpha - div beta grad) phi = RHS - p_mlabec->setBCoeffs(amrlev, amrex::GetArrOfConstPtrs(beta_face)); + // (div beta grad) phi = rhs + p_mlnode->setSigma(amrlev, beta_cc); //Declare MLMG object - pMLMG = std::make_unique(*p_mlabec); + pMLMG = std::make_unique(*p_mlnode); pMLMG->setVerbose(mlmg_verbosity); + #endif // INITIALIZE P in FE and rho in SC regions //InitializePandRho(P_old, Gamma, charge_den, e_den, hole_den, geom, prob_lo, prob_hi);//old - InitializePandRho(P_old, Gamma, charge_den, e_den, hole_den, MaterialMask, tphaseMask, n_cell, geom, prob_lo, prob_hi);//mask based + InitializePandRho(P_old, Gamma, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, MaterialMask, tphaseMask, n_cell, geom, prob_lo, prob_hi);//mask based //Obtain self consisten Phi and rho Real tol = 1.e-5; @@ -267,26 +317,28 @@ void main_main (c_FerroX& rFerroX) while(err > tol){ //Compute RHS of Poisson equation - ComputePoissonRHS(PoissonRHS, P_old, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + ComputePoissonRHS(Nodal_PoissonRHS, P_old, Nodal_charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); - dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); + //dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); - ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); + //ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); -#ifdef AMREX_USE_EB - p_mlebabec->setACoeffs(0, alpha_cc); -#else - p_mlabec->setACoeffs(0, alpha_cc); -#endif +//#ifdef AMREX_USE_EB +// p_mlebabec->setACoeffs(0, alpha_cc); +//#else +// p_mlabec->setACoeffs(0, alpha_cc); +//#endif //Initial guess for phi - PoissonPhi.setVal(0.); + Nodal_PoissonPhi.setVal(0.); //Poisson Solve - pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); - PoissonPhi.FillBoundary(geom.periodicity()); + pMLMG->solve({&Nodal_PoissonPhi}, {&Nodal_PoissonRHS}, 1.e-10, -1); + Nodal_PoissonPhi.FillBoundary(geom.periodicity()); // Calculate rho from Phi in SC region - ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); + //ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); + //Nodal + ComputeRho(Nodal_PoissonPhi, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, MaterialMask); if (contains_SC == 0) { // no semiconductor region; set error to zero so the while loop terminates @@ -295,13 +347,13 @@ void main_main (c_FerroX& rFerroX) // Calculate Error if (iter > 0){ - MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); - MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); - err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); + MultiFab::Copy(Nodal_PhiErr, Nodal_PoissonPhi, 0, 0, 1, 0); + MultiFab::Subtract(Nodal_PhiErr, Nodal_PoissonPhi_Prev, 0, 0, 1, 0); + err = Nodal_PhiErr.norm1(0, geom.periodicity())/Nodal_PoissonPhi.norm1(0, geom.periodicity()); } //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration - MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); + MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 0); iter = iter + 1; amrex::Print() << iter << " iterations :: err = " << err << std::endl; @@ -311,8 +363,10 @@ void main_main (c_FerroX& rFerroX) amrex::Print() << "\n ========= Self-Consistent Initialization of P and Rho Done! ========== \n"<< iter << " iterations to obtain self consistent Phi with err = " << err << std::endl; // Calculate E from Phi - ComputeEfromPhi(PoissonPhi, E, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); + ComputeEfromPhi(Nodal_PoissonPhi, E, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); + amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); + amrex::average_node_to_cellcenter(PoissonRHS, 0, Nodal_PoissonRHS, 0, 1); // Write a plotfile of the initial data if plot_int > 0 if (plot_int > 0) { @@ -383,27 +437,29 @@ void main_main (c_FerroX& rFerroX) while(err > tol){ // Compute RHS of Poisson equation - ComputePoissonRHS(PoissonRHS, P_new_pre, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + ComputePoissonRHS(Nodal_PoissonRHS, P_new_pre, Nodal_charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); - dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_new_pre, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); + //dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_new_pre, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); - ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); + //ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); -#ifdef AMREX_USE_EB - p_mlebabec->setACoeffs(0, alpha_cc); -#else - p_mlabec->setACoeffs(0, alpha_cc); -#endif +//#ifdef AMREX_USE_EB +// p_mlebabec->setACoeffs(0, alpha_cc); +//#else +// p_mlabec->setACoeffs(0, alpha_cc); +//#endif //Initial guess for phi - PoissonPhi.setVal(0.); + Nodal_PoissonPhi.setVal(0.); //Poisson Solve - pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); + pMLMG->solve({&Nodal_PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); - PoissonPhi.FillBoundary(geom.periodicity()); + Nodal_PoissonPhi.FillBoundary(geom.periodicity()); // Calculate rho from Phi in SC region - ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); + //ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); + //Nodal + ComputeRho(Nodal_PoissonPhi, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, MaterialMask); if (contains_SC == 0) { // no semiconductor region; set error to zero so the while loop terminates @@ -412,13 +468,13 @@ void main_main (c_FerroX& rFerroX) // Calculate Error if (iter > 0){ - MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); - MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); - err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); + MultiFab::Copy(Nodal_PhiErr, Nodal_PoissonPhi, 0, 0, 1, 0); + MultiFab::Subtract(Nodal_PhiErr, Nodal_PoissonPhi_Prev, 0, 0, 1, 0); + err = Nodal_PhiErr.norm1(0, geom.periodicity())/Nodal_PoissonPhi.norm1(0, geom.periodicity()); } //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration - MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); + MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 0); iter = iter + 1; amrex::Print() << iter << " iterations :: err = " << err << std::endl; @@ -453,27 +509,29 @@ void main_main (c_FerroX& rFerroX) while(err > tol){ // Compute RHS of Poisson equation - ComputePoissonRHS(PoissonRHS, P_new, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + ComputePoissonRHS(Nodal_PoissonRHS, P_new, Nodal_charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); - dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_new, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); + //dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_new, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); - ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); + //ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); -#ifdef AMREX_USE_EB - p_mlebabec->setACoeffs(0, alpha_cc); -#else - p_mlabec->setACoeffs(0, alpha_cc); -#endif +//#ifdef AMREX_USE_EB +// p_mlebabec->setACoeffs(0, alpha_cc); +//#else +// p_mlabec->setACoeffs(0, alpha_cc); +//#endif //Initial guess for phi - PoissonPhi.setVal(0.); + Nodal_PoissonPhi.setVal(0.); //Poisson Solve - pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); - PoissonPhi.FillBoundary(geom.periodicity()); + pMLMG->solve({&Nodal_PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); + Nodal_PoissonPhi.FillBoundary(geom.periodicity()); // Calculate rho from Phi in SC region - ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); + //ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); + //Nodal + ComputeRho(Nodal_PoissonPhi, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, MaterialMask); if (contains_SC == 0) { // no semiconductor region; set error to zero so the while loop terminates @@ -482,13 +540,13 @@ void main_main (c_FerroX& rFerroX) // Calculate Error if (iter > 0){ - MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); - MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); - err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); + MultiFab::Copy(Nodal_PhiErr, Nodal_PoissonPhi, 0, 0, 1, 0); + MultiFab::Subtract(Nodal_PhiErr, Nodal_PoissonPhi_Prev, 0, 0, 1, 0); + err = Nodal_PhiErr.norm1(0, geom.periodicity())/Nodal_PoissonPhi.norm1(0, geom.periodicity()); } //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration - MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); + MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 0); iter = iter + 1; amrex::Print() << iter << " iterations :: err = " << err << std::endl; @@ -507,14 +565,14 @@ void main_main (c_FerroX& rFerroX) //MultiFab::Copy(Phidiff, PoissonPhi, 0, 0, 1, 0); //MultiFab::Subtract(Phidiff, PoissonPhi_Old, 0, 0, 1, 0); - Real phi_max = PoissonPhi_Old.norm0(); + Real phi_max = Nodal_PoissonPhi_Old.norm0(); - for (MFIter mfi(PoissonPhi); mfi.isValid(); ++mfi) + for (MFIter mfi(Nodal_PoissonPhi); mfi.isValid(); ++mfi) { const Box& bx = mfi.growntilebox(1); - const Array4& Phi = PoissonPhi.array(mfi); - const Array4& PhiOld = PoissonPhi_Old.array(mfi); + const Array4& Phi = Nodal_PoissonPhi.array(mfi); + const Array4& PhiOld = Nodal_PoissonPhi_Old.array(mfi); const Array4& Phi_err = Phidiff.array(mfi); @@ -532,13 +590,13 @@ void main_main (c_FerroX& rFerroX) } //Copy PoissonPhi to PoissonPhi_Old to calculate difference at the next iteration - MultiFab::Copy(PoissonPhi_Old, PoissonPhi, 0, 0, 1, 0); + MultiFab::Copy(Nodal_PoissonPhi_Old, Nodal_PoissonPhi, 0, 0, 1, 0); amrex::Print() << "Steady state check : (phi(t) - phi(t-1)).norm0() = " << max_phi_err << std::endl; // Calculate E from Phi - ComputeEfromPhi(PoissonPhi, E, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); + ComputeEfromPhi(Nodal_PoissonPhi, E, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); Real step_stop_time = ParallelDescriptor::second() - step_strt_time; @@ -551,6 +609,8 @@ void main_main (c_FerroX& rFerroX) time = time + dt; + amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); + amrex::average_node_to_cellcenter(PoissonRHS, 0, Nodal_PoissonRHS, 0, 1); // Write a plotfile of the current data (plot_int was defined in the inputs file) if (plot_int > 0 && (step%plot_int == 0 || step == steady_state_step)) { @@ -591,14 +651,14 @@ void main_main (c_FerroX& rFerroX) amrex::Print() << "step = " << step << ", Phi_Bc_hi = " << Phi_Bc_hi << std::endl; // Set Dirichlet BC for Phi in z - SetPhiBC_z(PoissonPhi, n_cell); + SetPhiBC_z(Nodal_PoissonPhi, n_cell); // set Dirichlet BC by reading in the ghost cell values -#ifdef AMREX_USE_EB - p_mlebabec->setLevelBC(amrlev, &PoissonPhi); -#else - p_mlabec->setLevelBC(amrlev, &PoissonPhi); -#endif +//#ifdef AMREX_USE_EB +// p_mlebabec->setLevelBC(amrlev, &PoissonPhi); +//#else +// p_mlabec->setLevelBC(amrlev, &PoissonPhi); +//#endif err = 1.0; iter = 0; @@ -607,27 +667,29 @@ void main_main (c_FerroX& rFerroX) while(err > tol){ // Compute RHS of Poisson equation - ComputePoissonRHS(PoissonRHS, P_old, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + ComputePoissonRHS(Nodal_PoissonRHS, P_old, Nodal_charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); - dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); + //dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); - ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); + //ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); -#ifdef AMREX_USE_EB - p_mlebabec->setACoeffs(0, alpha_cc); -#else - p_mlabec->setACoeffs(0, alpha_cc); -#endif +//#ifdef AMREX_USE_EB +// p_mlebabec->setACoeffs(0, alpha_cc); +//#else +// p_mlabec->setACoeffs(0, alpha_cc); +//#endif //Initial guess for phi - PoissonPhi.setVal(0.); + Nodal_PoissonPhi.setVal(0.); //Poisson Solve - pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); - PoissonPhi.FillBoundary(geom.periodicity()); + pMLMG->solve({&Nodal_PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); + Nodal_PoissonPhi.FillBoundary(geom.periodicity()); // Calculate rho from Phi in SC region - ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); + //ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); + //Nodal + ComputeRho(Nodal_PoissonPhi, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, MaterialMask); if (contains_SC == 0) { // no semiconductor region; set error to zero so the while loop terminates @@ -636,13 +698,13 @@ void main_main (c_FerroX& rFerroX) // Calculate Error if (iter > 0){ - MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); - MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); - err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); + MultiFab::Copy(Nodal_PhiErr, Nodal_PoissonPhi, 0, 0, 1, 0); + MultiFab::Subtract(Nodal_PhiErr, Nodal_PoissonPhi_Prev, 0, 0, 1, 0); + err = Nodal_PhiErr.norm1(0, geom.periodicity())/Nodal_PoissonPhi.norm1(0, geom.periodicity()); } //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration - MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); + MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 0); iter = iter + 1; amrex::Print() << iter << " iterations :: err = " << err << std::endl; From 9dad5a7d06e1d667094c164b96f87affda9844f5 Mon Sep 17 00:00:00 2001 From: prkkumar Date: Thu, 29 Jun 2023 16:53:08 -0700 Subject: [PATCH 03/22] fix bugs --- Source/Solver/DerivativeAlgorithm.H | 205 +++++++++++++++++++++++++- Source/Solver/ElectrostaticSolver.cpp | 23 +-- Source/main.cpp | 14 +- 3 files changed, 223 insertions(+), 19 deletions(-) diff --git a/Source/Solver/DerivativeAlgorithm.H b/Source/Solver/DerivativeAlgorithm.H index ccc2e76..0176ca8 100644 --- a/Source/Solver/DerivativeAlgorithm.H +++ b/Source/Solver/DerivativeAlgorithm.H @@ -55,7 +55,7 @@ using namespace FerroX; /** * Perform first derivative dP/dx */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - static amrex::Real DPDx ( + static amrex::Real NodalDPDx ( amrex::Array4 const& F, amrex::Array4 const& mask, int const i, int const j, int const k, amrex::GpuArray dx @@ -123,10 +123,76 @@ using namespace FerroX; )/4./dx[0]; } +/** + * Perform first derivative dP/dx */ + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static amrex::Real DPDx ( + amrex::Array4 const& F, + amrex::Array4 const& mask, + int const i, int const j, int const k, amrex::GpuArray dx +) { + if (mask(i-1,j,k) != 0.0 && mask(i,j,k) == 0.0) { //FE lower boundary + + if(P_BC_flag_lo[0] == 0){ + Real F_lo = 0.0; + return (-4.*F_lo + 3.*F(i,j,k) + F(i+1,j,k))/(3.*dx[0]);//2nd order using three point stencil using 0, pOld(i,j,k), and pOld(i+1,j,k) + + } else if (P_BC_flag_lo[0] == 1){ + + Real F_lo = F(i,j,k)/(1 + dx[0]/2/lambda); + return (dx[0]*F_lo/lambda - F(i,j,k) + F(i+1,j,k))/(2.*dx[0]); // dP/dz = P_lo/lambda; + + // Real F_lo = (9. * F(i,j,k) - F(i,j,k+1)) / (3. * dx[2] / lambda + 8.); // derived with 2nd order one-sided 1st derivative + // return -(dx[2]*F_lo/lambda - F(i,j,k) + F(i,j,k+1))/(2.*dx[2]);// dP/dz = P_lo/lambda; + + } else if (P_BC_flag_lo[0] == 2){ + return ( - F(i,j,k) + F(i+1,j,k))/(2.*dx[0]); //dPdx = 0. + + } else if (P_BC_flag_lo[0] == 3){ + return (- F(i-1,j,k) + F(i+1,j,k))/(2.*dx[0]); //periodic + + } else { + amrex::Abort("Wrong flag of the lower x polarization boundary condition!!"); + return 0.0; + } + + } else if (mask(i+1,j,k) != 0.0 && mask(i,j,k) == 0.0){ // FE higher boundary + + if(P_BC_flag_hi[0] == 0){ + Real F_hi = 0.0; + return (4.*F_hi - 3.*F(i,j,k) - F(i-1,j,k))/(3.*dx[0]);//2nd order using three point stencil using 0, pOld(i,j,k), and pOld(i-1,j,k) + + } else if (P_BC_flag_hi[0] == 1){ + + Real F_hi = F(i,j,k)/(1 - dx[0]/2/lambda); + return (dx[0]*F_hi/lambda + F(i,j,k) - F(i-1,j,k))/(2.*dx[0]);//dPdz = P_hi/lambda; + + // Real F_hi = (9. * F(i,j,k) - F(i,j,k-1)) / ( - 3. * dx[2] / lambda + 8.); // derived with 2nd order one-sided 1st derivative + // return -(dx[2]*F_hi/lambda + F(i,j,k) - F(i,j,k-1))/(2.*dx[2]);//dPdz = P_hi/lambda; + + } else if (P_BC_flag_hi[0] == 2){ + return (F(i,j,k) - F(i-1,j,k))/(2.*dx[0]); //dPdx = 0. + + } else if (P_BC_flag_hi[0] == 3){ + return (F(i+1,j,k) - F(i-1,j,k))/(2.*dx[0]); //Periodic + + } else { + amrex::Abort("Wrong flag of the higher x polarization boundary condition!!"); + return 0.0; + } + + } else if (mask(i,j,k) == 0.0) { // inside FE + return (F(i+1,j,k) - F(i-1,j,k))/(2.*dx[0]); + + } else { + return 0.0; + } + } + /** * Perform first derivative dP/dy */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - static amrex::Real DPDy ( + static amrex::Real NodalDPDy ( amrex::Array4 const& F, amrex::Array4 const& mask, int const i, int const j, int const k, amrex::GpuArray dx @@ -195,10 +261,77 @@ using namespace FerroX; )/4./dx[1]; } +/** + * Perform first derivative dP/dy */ + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static amrex::Real DPDy ( + amrex::Array4 const& F, + amrex::Array4 const& mask, + int const i, int const j, int const k, amrex::GpuArray dx +) { + + if (mask(i,j-1,k) != 0.0 && mask(i,j,k) == 0.0) { //FE lower boundary + + if(P_BC_flag_lo[1] == 0){ + Real F_lo = 0.0; + return (-4.*F_lo + 3.*F(i,j,k) + F(i,j+1,k))/(3.*dx[1]);//2nd order using three point stencil using 0, pOld(i,j,k), and pOld(i,j,k+1) + + } else if (P_BC_flag_lo[1] == 1){ + + Real F_lo = F(i,j,k)/(1 + dx[1]/2/lambda); + return (dx[1]*F_lo/lambda - F(i,j,k) + F(i,j+1,k))/(2.*dx[1]); // dP/dz = P_lo/lambda; + + // Real F_lo = (9. * F(i,j,k) - F(i,j,k+1)) / (3. * dx[2] / lambda + 8.); // derived with 2nd order one-sided 1st derivative + // return -(dx[2]*F_lo/lambda - F(i,j,k) + F(i,j,k+1))/(2.*dx[2]);// dP/dz = P_lo/lambda; + + } else if (P_BC_flag_lo[1] == 2){ + return ( - F(i,j,k) + F(i,j+1,k))/(2.*dx[1]); //dPdy = 0. + + } else if (P_BC_flag_lo[1] == 3){ + return ( - F(i,j-1,k) + F(i,j+1,k))/(2.*dx[1]); //Periodic + + } else { + amrex::Abort("Wrong flag of the lower polarization boundary condition!!"); + return 0.0; + } + + } else if (mask(i,j+1,k) != 0.0 && mask(i,j,k) == 0.0){ // FE higher boundary + + if(P_BC_flag_hi[1] == 0){ + Real F_hi = 0.0; + return (4.*F_hi - 3.*F(i,j,k) - F(i,j-1,k))/(3.*dx[1]);//2nd order using three point stencil using 0, pOld(i,j,k), and pOld(i,j,k-1) + + } else if (P_BC_flag_hi[1] == 1){ + + Real F_hi = F(i,j,k)/(1 - dx[1]/2/lambda); + return (dx[1]*F_hi/lambda + F(i,j,k) - F(i,j-1,k))/(2.*dx[1]);//dPdz = P_hi/lambda; + + // Real F_hi = (9. * F(i,j,k) - F(i,j,k-1)) / ( - 3. * dx[2] / lambda + 8.); // derived with 2nd order one-sided 1st derivative + // return -(dx[2]*F_hi/lambda + F(i,j,k) - F(i,j,k-1))/(2.*dx[2]);//dPdz = P_hi/lambda; + + } else if (P_BC_flag_hi[1] == 2){ + return (F(i,j,k) - F(i,j-1,k))/(2.*dx[1]); //dPdy = 0. + + } else if (P_BC_flag_hi[1] == 3){ + return (F(i,j+1,k) - F(i,j-1,k))/(2.*dx[1]); //Periodic + + } else { + amrex::Abort("Wrong flag of the higher y polarization boundary condition!!"); + return 0.0; + } + + } else if (mask(i,j,k) == 0.0) { // inside FE + return (F(i,j+1,k) - F(i,j-1,k))/(2.*dx[1]); + + } else { + return 0.0; + } + } + /** * Perform first derivative dP/dz */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - static amrex::Real DPDz ( + static amrex::Real NodalDPDz ( amrex::Array4 const& F, amrex::Array4 const& mask, int const i, int const j, int const k, amrex::GpuArray dx @@ -266,6 +399,72 @@ using namespace FerroX; + F(i, j, k) - F(i, j, k-1) )/4./dx[2]; } +/** + * Perform first derivative dP/dz */ + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + static amrex::Real DPDz ( + amrex::Array4 const& F, + amrex::Array4 const& mask, + int const i, int const j, int const k, amrex::GpuArray dx +) { + + if (mask(i,j,k-1) != 0.0 && mask(i,j,k) == 0.0) { //FE lower boundary + + if(P_BC_flag_lo[2] == 0){ + Real F_lo = 0.0; + return (-4.*F_lo + 3.*F(i,j,k) + F(i,j,k+1))/(3.*dx[2]);//2nd order using three point stencil using 0, pOld(i,j,k), and pOld(i,j,k+1) + + } else if (P_BC_flag_lo[2] == 1){ + + Real F_lo = F(i,j,k)/(1 + dx[2]/2/lambda); + return (dx[2]*F_lo/lambda - F(i,j,k) + F(i,j,k+1))/(2.*dx[2]); // dP/dz = P_lo/lambda; + + // Real F_lo = (9. * F(i,j,k) - F(i,j,k+1)) / (3. * dx[2] / lambda + 8.); // derived with 2nd order one-sided 1st derivative + // return -(dx[2]*F_lo/lambda - F(i,j,k) + F(i,j,k+1))/(2.*dx[2]);// dP/dz = P_lo/lambda; + + } else if (P_BC_flag_lo[2] == 2){ + return ( - F(i,j,k) + F(i,j,k+1))/(2.*dx[2]); //dPdz = 0. + + } else if (P_BC_flag_lo[2] == 3){ + return ( - F(i,j,k-1) + F(i,j,k+1))/(2.*dx[2]); //No BC + + } else { + amrex::Abort("Wrong flag of the lower polarization boundary condition!!"); + return 0.0; + } + + } else if ( mask(i,j,k+1) != 0.0 && mask(i,j,k) == 0.0 ){ // FE higher boundary + + if(P_BC_flag_hi[2] == 0){ + Real F_hi = 0.0; + return (4.*F_hi - 3.*F(i,j,k) - F(i,j,k-1))/(3.*dx[2]);//2nd order using three point stencil using 0, pOld(i,j,k), and pOld(i,j,k-1) + + } else if (P_BC_flag_hi[2] == 1){ + + Real F_hi = F(i,j,k)/(1 - dx[2]/2/lambda); + return (dx[2]*F_hi/lambda + F(i,j,k) - F(i,j,k-1))/(2.*dx[2]);//dPdz = P_hi/lambda; + + // Real F_hi = (9. * F(i,j,k) - F(i,j,k-1)) / ( - 3. * dx[2] / lambda + 8.); // derived with 2nd order one-sided 1st derivative + // return -(dx[2]*F_hi/lambda + F(i,j,k) - F(i,j,k-1))/(2.*dx[2]);//dPdz = P_hi/lambda; + + } else if (P_BC_flag_hi[2] == 2){ + return (F(i,j,k) - F(i,j,k-1))/(2.*dx[2]); //dPdz = 0. + + } else if (P_BC_flag_hi[2] == 3){ + return (0. - F(i,j,k-1))/(2.*dx[2]); //No BC + + } else { + amrex::Abort("Wrong flag of the higher polarization boundary condition!!"); + return 0.0; + } + + } else if (mask(i,j,k) == 0.0) { // inside FE + return (F(i,j,k+1) - F(i,j,k-1))/(2.*dx[2]); + + } else { + return 0.0; + } + } // ///** // * Perform double derivative (d^2)P/dx^2 */ diff --git a/Source/Solver/ElectrostaticSolver.cpp b/Source/Solver/ElectrostaticSolver.cpp index 486ca6d..2007a52 100644 --- a/Source/Solver/ElectrostaticSolver.cpp +++ b/Source/Solver/ElectrostaticSolver.cpp @@ -32,9 +32,9 @@ void ComputePoissonRHS(MultiFab& PoissonRHS, //Convert Euler angles from degrees to radians amrex::Real Pi = 3.14159265358979323846; - amrex::Real alpha_rad = Pi/180.*angle_alpha_arr(i,j,k); - amrex::Real beta_rad = Pi/180.*angle_beta_arr(i,j,k); - amrex::Real theta_rad = Pi/180.*angle_theta_arr(i,j,k); + amrex::Real alpha_rad = 45.*Pi/180.;//Pi/180.*angle_alpha_arr(i,j,k); + amrex::Real beta_rad = 45.*Pi/180.;//Pi/180.*angle_beta_arr(i,j,k); + amrex::Real theta_rad = 45.*Pi/180.;//Pi/180.*angle_theta_arr(i,j,k); amrex::Real R_11, R_12, R_13, R_21, R_22, R_23, R_31, R_32, R_33; @@ -63,16 +63,19 @@ void ComputePoissonRHS(MultiFab& PoissonRHS, if(mask(i,j,k) >= 2.0){ //SC region RHS(i,j,k) = charge_den_arr(i,j,k); + RHS(i,j,k) *= -1.; } else if(mask(i,j,k) == 1.0){ //DE region RHS(i,j,k) = 0.; } else { //mask(i,j,k) == 0.0 FE region - RHS(i,j,k) = - (R_11*DPDx(pOld_p, mask, i, j, k, dx) + R_12*DPDy(pOld_p, mask, i, j, k, dx) + R_13*DPDz(pOld_p, mask, i, j, k, dx)) - - (R_21*DPDx(pOld_q, mask, i, j, k, dx) + R_22*DPDy(pOld_q, mask, i, j, k, dx) + R_23*DPDz(pOld_q, mask, i, j, k, dx)) - - (R_31*DPDx(pOld_r, mask, i, j, k, dx) + R_32*DPDy(pOld_r, mask, i, j, k, dx) + R_33*DPDz(pOld_r, mask, i, j, k, dx)); + RHS(i,j,k) = - (R_11*NodalDPDx(pOld_p, mask, i, j, k, dx) + R_12*NodalDPDy(pOld_p, mask, i, j, k, dx) + R_13*NodalDPDz(pOld_p, mask, i, j, k, dx)) + - (R_21*NodalDPDx(pOld_q, mask, i, j, k, dx) + R_22*NodalDPDy(pOld_q, mask, i, j, k, dx) + R_23*NodalDPDz(pOld_q, mask, i, j, k, dx)) + - (R_31*NodalDPDx(pOld_r, mask, i, j, k, dx) + R_32*NodalDPDy(pOld_r, mask, i, j, k, dx) + R_33*NodalDPDz(pOld_r, mask, i, j, k, dx)); + RHS(i,j,k) *= -1.; +// RHS(i,j,k) = -0.; } }); @@ -138,7 +141,7 @@ void ComputeEfromPhi(MultiFab& PoissonPhi, { // Calculate E from Phi - for ( MFIter mfi(PoissonPhi); mfi.isValid(); ++mfi ) + for ( MFIter mfi(E[0]); mfi.isValid(); ++mfi ) { const Box& bx = mfi.validbox(); @@ -162,9 +165,9 @@ void ComputeEfromPhi(MultiFab& PoissonPhi, //Convert Euler angles from degrees to radians amrex::Real Pi = 3.14159265358979323846; - amrex::Real alpha_rad = Pi/180.*angle_alpha_arr(i,j,k); - amrex::Real beta_rad = Pi/180.*angle_beta_arr(i,j,k); - amrex::Real theta_rad = Pi/180.*angle_theta_arr(i,j,k); + amrex::Real alpha_rad = 45.*Pi/180.;//Pi/180.*angle_alpha_arr(i,j,k); + amrex::Real beta_rad = 45.*Pi/180.;//Pi/180.*angle_beta_arr(i,j,k); + amrex::Real theta_rad = 45.*Pi/180.;//Pi/180.*angle_theta_arr(i,j,k); amrex::Real R_11, R_12, R_13, R_21, R_22, R_23, R_31, R_32, R_33; diff --git a/Source/main.cpp b/Source/main.cpp index 09fed8a..c47df4d 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -452,7 +452,7 @@ void main_main (c_FerroX& rFerroX) Nodal_PoissonPhi.setVal(0.); //Poisson Solve - pMLMG->solve({&Nodal_PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); + pMLMG->solve({&Nodal_PoissonPhi}, {&Nodal_PoissonRHS}, 1.e-10, -1); Nodal_PoissonPhi.FillBoundary(geom.periodicity()); @@ -525,7 +525,7 @@ void main_main (c_FerroX& rFerroX) Nodal_PoissonPhi.setVal(0.); //Poisson Solve - pMLMG->solve({&Nodal_PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); + pMLMG->solve({&Nodal_PoissonPhi}, {&Nodal_PoissonRHS}, 1.e-10, -1); Nodal_PoissonPhi.FillBoundary(geom.periodicity()); // Calculate rho from Phi in SC region @@ -573,7 +573,7 @@ void main_main (c_FerroX& rFerroX) const Array4& Phi = Nodal_PoissonPhi.array(mfi); const Array4& PhiOld = Nodal_PoissonPhi_Old.array(mfi); - const Array4& Phi_err = Phidiff.array(mfi); + const Array4& Phi_err = Nodal_Phidiff.array(mfi); amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) @@ -582,11 +582,13 @@ void main_main (c_FerroX& rFerroX) }); } - Real max_phi_err = Phidiff.norm0(); + Real max_phi_err = Nodal_Phidiff.norm0(); - if (max_phi_err < phi_tolerance) { + if(step > 1){ + if (max_phi_err < phi_tolerance) { steady_state_step = step; inc_step = step; + } } //Copy PoissonPhi to PoissonPhi_Old to calculate difference at the next iteration @@ -683,7 +685,7 @@ void main_main (c_FerroX& rFerroX) Nodal_PoissonPhi.setVal(0.); //Poisson Solve - pMLMG->solve({&Nodal_PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); + pMLMG->solve({&Nodal_PoissonPhi}, {&Nodal_PoissonRHS}, 1.e-10, -1); Nodal_PoissonPhi.FillBoundary(geom.periodicity()); // Calculate rho from Phi in SC region From 2b662667447fe7c29cd79eee1d94f774c3def7ed Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Fri, 30 Jun 2023 11:12:46 -0700 Subject: [PATCH 04/22] tweaks for nonhomogenous dirichlet BC --- Source/Solver/ElectrostaticSolver.H | 2 ++ Source/Solver/ElectrostaticSolver.cpp | 23 ++++++++++++++++------- Source/main.cpp | 8 +++++++- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/Source/Solver/ElectrostaticSolver.H b/Source/Solver/ElectrostaticSolver.H index 592c99e..e914e2f 100644 --- a/Source/Solver/ElectrostaticSolver.H +++ b/Source/Solver/ElectrostaticSolver.H @@ -55,6 +55,8 @@ void ComputePoissonRHS_Newton(MultiFab& PoissonRHS, MultiFab& PoissonPhi, MultiFab& alpha_cc); +void ComputeAx_H(MultiFab& x_H, MultiFab& Ax_H, const amrex::GpuArray& n_cell); + void SetPhiBC_z(MultiFab& PossonPhi, const amrex::GpuArray& n_cell); void SetPoissonBC(c_FerroX& rFerroX, std::array,2>& LinOpBCType_2d, bool& all_homogeneous_boundaries, bool& some_functionbased_inhomogeneous_boundaries, bool& some_constant_inhomogeneous_boundaries); diff --git a/Source/Solver/ElectrostaticSolver.cpp b/Source/Solver/ElectrostaticSolver.cpp index 2007a52..79e05e5 100644 --- a/Source/Solver/ElectrostaticSolver.cpp +++ b/Source/Solver/ElectrostaticSolver.cpp @@ -32,9 +32,9 @@ void ComputePoissonRHS(MultiFab& PoissonRHS, //Convert Euler angles from degrees to radians amrex::Real Pi = 3.14159265358979323846; - amrex::Real alpha_rad = 45.*Pi/180.;//Pi/180.*angle_alpha_arr(i,j,k); + amrex::Real alpha_rad = 0.*Pi/180.;//Pi/180.*angle_alpha_arr(i,j,k); amrex::Real beta_rad = 45.*Pi/180.;//Pi/180.*angle_beta_arr(i,j,k); - amrex::Real theta_rad = 45.*Pi/180.;//Pi/180.*angle_theta_arr(i,j,k); + amrex::Real theta_rad = 0.*Pi/180.;//Pi/180.*angle_theta_arr(i,j,k); amrex::Real R_11, R_12, R_13, R_21, R_22, R_23, R_31, R_32, R_33; @@ -165,9 +165,9 @@ void ComputeEfromPhi(MultiFab& PoissonPhi, //Convert Euler angles from degrees to radians amrex::Real Pi = 3.14159265358979323846; - amrex::Real alpha_rad = 45.*Pi/180.;//Pi/180.*angle_alpha_arr(i,j,k); + amrex::Real alpha_rad = 0.*Pi/180.;//Pi/180.*angle_alpha_arr(i,j,k); amrex::Real beta_rad = 45.*Pi/180.;//Pi/180.*angle_beta_arr(i,j,k); - amrex::Real theta_rad = 45.*Pi/180.;//Pi/180.*angle_theta_arr(i,j,k); + amrex::Real theta_rad = 0.*Pi/180.;//Pi/180.*angle_theta_arr(i,j,k); amrex::Real R_11, R_12, R_13, R_21, R_22, R_23, R_31, R_32, R_33; @@ -500,19 +500,28 @@ void Fill_FunctionBased_Inhomogeneous_Boundaries(c_FerroX& rFerroX, MultiFab& Po } } +//We would like to solve A x = b with inhomogeneous bc's +//Here, "A" is div sigma grad +//This is equivalent to A_H x = b - A x_H, where +//A is the inhomogeneous operator +//A_H is the homogeneous operator +//x_H is a multifab filled with zeros, but ghost cells filled to respect bc's + + void SetPhiBC_z(MultiFab& PoissonPhi, const amrex::GpuArray& n_cell) { + PoissonPhi.setVal(0.); for (MFIter mfi(PoissonPhi); mfi.isValid(); ++mfi) { - const Box& bx = mfi.growntilebox(1); + const Box& bx = mfi.validbox(); const Array4& Phi = PoissonPhi.array(mfi); amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) { - if(k < 0) { + if(k == 0) { Phi(i,j,k) = Phi_Bc_lo; - } else if(k >= n_cell[2]){ + } else if(k == n_cell[2]){ Phi(i,j,k) = Phi_Bc_hi; } }); diff --git a/Source/main.cpp b/Source/main.cpp index c47df4d..4accef2 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -120,6 +120,8 @@ void main_main (c_FerroX& rFerroX) MultiFab Nodal_PoissonPhi(nba, dm, 1, 1); MultiFab Nodal_PoissonPhi_Old(nba, dm, 1, 1); MultiFab Nodal_PoissonPhi_Prev(nba, dm, 1, 1); + MultiFab Nodal_PoissonPhi_BC(nba, dm, 1, 1); + MultiFab APoissonPhi_BC(nba, dm, 1, 1); MultiFab Nodal_PhiErr(nba, dm, 1, 1); MultiFab Nodal_Phidiff(nba, dm, 1, 1); @@ -315,10 +317,14 @@ void main_main (c_FerroX& rFerroX) int iter = 0; while(err > tol){ - + + SetPhiBC_z(Nodal_PoissonPhi_BC, n_cell); + p_mlnode->Fapply (0, 4, APoissonPhi_BC, Nodal_PoissonPhi_BC); //Compute RHS of Poisson equation ComputePoissonRHS(Nodal_PoissonRHS, P_old, Nodal_charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); + //dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); //ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); From 8b6123bab9d974e2f18e1593f3783ca148ff636c Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Fri, 30 Jun 2023 11:18:34 -0700 Subject: [PATCH 05/22] add comments --- Source/main.cpp | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/Source/main.cpp b/Source/main.cpp index 4accef2..2869c12 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -318,11 +318,25 @@ void main_main (c_FerroX& rFerroX) while(err > tol){ - SetPhiBC_z(Nodal_PoissonPhi_BC, n_cell); - p_mlnode->Fapply (0, 4, APoissonPhi_BC, Nodal_PoissonPhi_BC); - //Compute RHS of Poisson equation +/* + We would like to solve A x = b with inhomogeneous bc's + Here, "A" is div sigma grad + This is equivalent to A_H x = b - A x_H, where + A is the inhomogeneous operator + A_H is the homogeneous operator + x_H is a multifab filled with zeros, but boundary cells cells filled to respect bc's +*/ + + //Compute x_H + SetPhiBC_z(Nodal_PoissonPhi_BC, n_cell); + + //Compute A x_H + p_mlnode->Fapply (0, 4, APoissonPhi_BC, Nodal_PoissonPhi_BC); //mglev = 4 is hard coded for now based on n_cell and number of boxes + + //Compute b ComputePoissonRHS(Nodal_PoissonRHS, P_old, Nodal_charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + //Compute b - A x_H MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); //dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); From 4a3d5efa82fba0fa9e01754ea18fd8ce06e6f1c7 Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Thu, 6 Jul 2023 09:39:09 -0700 Subject: [PATCH 06/22] nodal angle parser --- Source/Solver/ElectrostaticSolver.H | 4 ++ Source/Solver/ElectrostaticSolver.cpp | 74 +++++++++++++++++----- Source/Solver/Initialization.H | 1 + Source/Solver/Initialization.cpp | 88 +++++++++++++++++++++++++++ Source/main.cpp | 79 ++++++++++++++++++++---- 5 files changed, 219 insertions(+), 27 deletions(-) diff --git a/Source/Solver/ElectrostaticSolver.H b/Source/Solver/ElectrostaticSolver.H index e914e2f..b6be471 100644 --- a/Source/Solver/ElectrostaticSolver.H +++ b/Source/Solver/ElectrostaticSolver.H @@ -59,6 +59,10 @@ void ComputeAx_H(MultiFab& x_H, MultiFab& Ax_H, const amrex::GpuArray& n_cell); +void SetPhiBC_z_after_solve(MultiFab& PossonPhi, const amrex::GpuArray& n_cell); + +void average_cc_to_nodes(MultiFab& mf_nodal, const MultiFab& mf_cc, const Geometry& geom); + void SetPoissonBC(c_FerroX& rFerroX, std::array,2>& LinOpBCType_2d, bool& all_homogeneous_boundaries, bool& some_functionbased_inhomogeneous_boundaries, bool& some_constant_inhomogeneous_boundaries); void Fill_Constant_Inhomogeneous_Boundaries(c_FerroX& rFerroX, MultiFab& PoissonPhi); diff --git a/Source/Solver/ElectrostaticSolver.cpp b/Source/Solver/ElectrostaticSolver.cpp index 79e05e5..1eaee89 100644 --- a/Source/Solver/ElectrostaticSolver.cpp +++ b/Source/Solver/ElectrostaticSolver.cpp @@ -32,9 +32,13 @@ void ComputePoissonRHS(MultiFab& PoissonRHS, //Convert Euler angles from degrees to radians amrex::Real Pi = 3.14159265358979323846; - amrex::Real alpha_rad = 0.*Pi/180.;//Pi/180.*angle_alpha_arr(i,j,k); - amrex::Real beta_rad = 45.*Pi/180.;//Pi/180.*angle_beta_arr(i,j,k); - amrex::Real theta_rad = 0.*Pi/180.;//Pi/180.*angle_theta_arr(i,j,k); + +// amrex::Real alpha_rad = 0.*Pi/180.;//Pi/180.*angle_alpha_arr(i,j,k); +// amrex::Real beta_rad = 45.*Pi/180.;//Pi/180.*angle_beta_arr(i,j,k); +// amrex::Real theta_rad = 0.*Pi/180.;//Pi/180.*angle_theta_arr(i,j,k); + amrex::Real alpha_rad = Pi/180.*angle_alpha_arr(i,j,k); + amrex::Real beta_rad = Pi/180.*angle_beta_arr(i,j,k); + amrex::Real theta_rad = Pi/180.*angle_theta_arr(i,j,k); amrex::Real R_11, R_12, R_13, R_21, R_22, R_23, R_31, R_32, R_33; @@ -165,9 +169,12 @@ void ComputeEfromPhi(MultiFab& PoissonPhi, //Convert Euler angles from degrees to radians amrex::Real Pi = 3.14159265358979323846; - amrex::Real alpha_rad = 0.*Pi/180.;//Pi/180.*angle_alpha_arr(i,j,k); - amrex::Real beta_rad = 45.*Pi/180.;//Pi/180.*angle_beta_arr(i,j,k); - amrex::Real theta_rad = 0.*Pi/180.;//Pi/180.*angle_theta_arr(i,j,k); +// amrex::Real alpha_rad = 0.*Pi/180.;//Pi/180.*angle_alpha_arr(i,j,k); +// amrex::Real beta_rad = 45.*Pi/180.;//Pi/180.*angle_beta_arr(i,j,k); +// amrex::Real theta_rad = 0.*Pi/180.;//Pi/180.*angle_theta_arr(i,j,k); + amrex::Real alpha_rad = Pi/180.*angle_alpha_arr(i,j,k); + amrex::Real beta_rad = Pi/180.*angle_beta_arr(i,j,k); + amrex::Real theta_rad = Pi/180.*angle_theta_arr(i,j,k); amrex::Real R_11, R_12, R_13, R_21, R_22, R_23, R_31, R_32, R_33; @@ -500,14 +507,7 @@ void Fill_FunctionBased_Inhomogeneous_Boundaries(c_FerroX& rFerroX, MultiFab& Po } } -//We would like to solve A x = b with inhomogeneous bc's -//Here, "A" is div sigma grad -//This is equivalent to A_H x = b - A x_H, where -//A is the inhomogeneous operator -//A_H is the homogeneous operator -//x_H is a multifab filled with zeros, but ghost cells filled to respect bc's - - +//A multifab filled with zeros, but boundary cells filled to respect bc's void SetPhiBC_z(MultiFab& PoissonPhi, const amrex::GpuArray& n_cell) { PoissonPhi.setVal(0.); @@ -528,3 +528,49 @@ void SetPhiBC_z(MultiFab& PoissonPhi, const amrex::GpuArray } } + +//A multifab filled with zeros, but boundary cells filled to respect bc's +void SetPhiBC_z_after_solve(MultiFab& PoissonPhi, const amrex::GpuArray& n_cell) +{ + for (MFIter mfi(PoissonPhi); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.validbox(); + + const Array4& Phi = PoissonPhi.array(mfi); + + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) + { + if(k == 0) { + Phi(i,j,k) = Phi_Bc_lo; + } else if(k == n_cell[2]){ + Phi(i,j,k) = Phi_Bc_hi; + } + }); + } +} + +//Avergae cell-centered multifab to nodes +void average_cc_to_nodes(MultiFab& mf_nodal, const MultiFab& mf_cc, const Geometry& geom) +{ + for (MFIter mfi(mf_nodal); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.validbox(); + + const Array4& mf_cc_arr = mf_cc.array(mfi); + const Array4& mf_nodal_arr = mf_nodal.array(mfi); + + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) + { + mf_nodal_arr(i,j,k) = 1./8.*(mf_cc_arr(i-1, j-1, k-1) + + mf_cc_arr(i, j-1, k-1) + + mf_cc_arr(i-1, j, k-1) + + mf_cc_arr(i, j, k-1) + + mf_cc_arr(i-1, j-1, k ) + + mf_cc_arr(i, j-1, k ) + + mf_cc_arr(i-1, j, k ) + + mf_cc_arr(i, j, k )); + }); + } + mf_nodal.FillBoundary(geom.periodicity()); +} + diff --git a/Source/Solver/Initialization.H b/Source/Solver/Initialization.H index 8a929d5..ab5813e 100644 --- a/Source/Solver/Initialization.H +++ b/Source/Solver/Initialization.H @@ -27,4 +27,5 @@ void InitializeMaterialMask(MultiFab& MaterialMask, void InitializeMaterialMask(c_FerroX& rFerroX, const Geometry& geom, MultiFab& MaterialMask); void Initialize_tphase_Mask(c_FerroX& rFerroX, const Geometry& geom, MultiFab& tphaseMask); void Initialize_Euler_angles(c_FerroX& rFerroX, const Geometry& geom, MultiFab& angle_alpha, MultiFab& angle_beta, MultiFab& angle_theta); +void Initialize_nodal_Euler_angles(c_FerroX& rFerroX, const Geometry& geom, MultiFab& Nodal_angle_alpha, MultiFab& Nodal_angle_beta, MultiFab& Nodal_angle_theta); diff --git a/Source/Solver/Initialization.cpp b/Source/Solver/Initialization.cpp index 2100462..3217be0 100644 --- a/Source/Solver/Initialization.cpp +++ b/Source/Solver/Initialization.cpp @@ -396,3 +396,91 @@ void Initialize_Euler_angles(c_FerroX& rFerroX, const Geometry& geom, MultiFab& angle_theta.FillBoundary(geom.periodicity()); } +// initialization of Euler angles on the nodes +void Initialize_nodal_Euler_angles(c_FerroX& rFerroX, const Geometry& geom, MultiFab& Nodal_angle_alpha, MultiFab& Nodal_angle_beta, MultiFab& Nodal_angle_theta) +{ + auto& rGprop = rFerroX.get_GeometryProperties(); + Box const& domain = rGprop.geom.Domain(); + + const auto dx = rGprop.geom.CellSizeArray(); + const auto& real_box = rGprop.geom.ProbDomain(); + const auto iv_alpha = Nodal_angle_alpha.ixType().toIntVect(); + const auto iv_beta = Nodal_angle_beta.ixType().toIntVect(); + const auto iv_theta = Nodal_angle_theta.ixType().toIntVect(); + + for (MFIter mfi(Nodal_angle_alpha); mfi.isValid(); ++mfi) + { + const auto& alpha_arr = Nodal_angle_alpha.array(mfi); + const auto& beta_arr = Nodal_angle_beta.array(mfi); + const auto& theta_arr = Nodal_angle_theta.array(mfi); + const auto& bx = mfi.validbox(); + + std::string alpha_s; + std::unique_ptr alpha_parser; + std::string m_str_alpha_function; + + std::string beta_s; + std::unique_ptr beta_parser; + std::string m_str_beta_function; + + std::string theta_s; + std::unique_ptr theta_parser; + std::string m_str_theta_function; + + ParmParse pp_alpha("angle_alpha"); + + + if (pp_alpha.query("alpha_function(x,y,z)", m_str_alpha_function) ) { + alpha_s = "parse_alpha_function"; + } + + if (alpha_s == "parse_alpha_function") { + Store_parserString(pp_alpha, "alpha_function(x,y,z)", m_str_alpha_function); + alpha_parser = std::make_unique( + makeParser(m_str_alpha_function,{"x","y","z"})); + } + + ParmParse pp_beta("angle_beta"); + + + if (pp_beta.query("beta_function(x,y,z)", m_str_beta_function) ) { + beta_s = "parse_beta_function"; + } + + if (beta_s == "parse_beta_function") { + Store_parserString(pp_beta, "beta_function(x,y,z)", m_str_beta_function); + beta_parser = std::make_unique( + makeParser(m_str_beta_function,{"x","y","z"})); + } + + ParmParse pp_theta("angle_theta"); + + + if (pp_theta.query("theta_function(x,y,z)", m_str_theta_function) ) { + theta_s = "parse_theta_function"; + } + + if (theta_s == "parse_theta_function") { + Store_parserString(pp_theta, "theta_function(x,y,z)", m_str_theta_function); + theta_parser = std::make_unique( + makeParser(m_str_theta_function,{"x","y","z"})); + } + + const auto& macro_parser_alpha = alpha_parser->compile<3>(); + const auto& macro_parser_beta = beta_parser->compile<3>(); + const auto& macro_parser_theta = theta_parser->compile<3>(); + + amrex::ParallelFor(bx, + [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + eXstatic_MFab_Util::ConvertParserIntoMultiFab_3vars(i,j,k,dx,real_box,iv_alpha,macro_parser_alpha,alpha_arr); + eXstatic_MFab_Util::ConvertParserIntoMultiFab_3vars(i,j,k,dx,real_box,iv_beta, macro_parser_beta, beta_arr ); + eXstatic_MFab_Util::ConvertParserIntoMultiFab_3vars(i,j,k,dx,real_box,iv_theta,macro_parser_theta,theta_arr); + }); + + } + Nodal_angle_alpha.FillBoundary(geom.periodicity()); + Nodal_angle_beta.FillBoundary(geom.periodicity()); + Nodal_angle_theta.FillBoundary(geom.periodicity()); +} + diff --git a/Source/main.cpp b/Source/main.cpp index 2869c12..07bcfac 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -120,8 +120,8 @@ void main_main (c_FerroX& rFerroX) MultiFab Nodal_PoissonPhi(nba, dm, 1, 1); MultiFab Nodal_PoissonPhi_Old(nba, dm, 1, 1); MultiFab Nodal_PoissonPhi_Prev(nba, dm, 1, 1); - MultiFab Nodal_PoissonPhi_BC(nba, dm, 1, 1); - MultiFab APoissonPhi_BC(nba, dm, 1, 1); + MultiFab Nodal_PoissonPhi_BC(nba, dm, 1, 0); + MultiFab APoissonPhi_BC(nba, dm, 1, 0); MultiFab Nodal_PhiErr(nba, dm, 1, 1); MultiFab Nodal_Phidiff(nba, dm, 1, 1); @@ -144,12 +144,22 @@ void main_main (c_FerroX& rFerroX) MultiFab angle_beta(ba, dm, 1, 0); MultiFab angle_theta(ba, dm, 1, 0); + //Nodal angles + MultiFab Nodal_angle_alpha(nba, dm, 1, 0); + MultiFab Nodal_angle_beta(nba, dm, 1, 0); + MultiFab Nodal_angle_theta(nba, dm, 1, 0); + //Initialize material mask InitializeMaterialMask(MaterialMask, geom, prob_lo, prob_hi); //InitializeMaterialMask(rFerroX, geom, MaterialMask); if(Coordinate_Transformation == 1){ Initialize_tphase_Mask(rFerroX, geom, tphaseMask); - Initialize_Euler_angles(rFerroX, geom, angle_alpha, angle_beta, angle_theta); + //Initialize_Euler_angles(rFerroX, geom, angle_alpha, angle_beta, angle_theta); + //average_cc_to_nodes(Nodal_angle_alpha, angle_alpha, geom); + //average_cc_to_nodes(Nodal_angle_beta, angle_beta, geom); + //average_cc_to_nodes(Nodal_angle_theta, angle_theta, geom); + + Initialize_nodal_Euler_angles(rFerroX, geom, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta); } else { tphaseMask.setVal(0.); } @@ -324,17 +334,17 @@ void main_main (c_FerroX& rFerroX) This is equivalent to A_H x = b - A x_H, where A is the inhomogeneous operator A_H is the homogeneous operator - x_H is a multifab filled with zeros, but boundary cells cells filled to respect bc's + x_H is a multifab filled with zeros, but boundary cells filled to respect bc's */ //Compute x_H SetPhiBC_z(Nodal_PoissonPhi_BC, n_cell); //Compute A x_H - p_mlnode->Fapply (0, 4, APoissonPhi_BC, Nodal_PoissonPhi_BC); //mglev = 4 is hard coded for now based on n_cell and number of boxes + pMLMG->apply ({&APoissonPhi_BC}, {&Nodal_PoissonPhi_BC}); //Compute b - ComputePoissonRHS(Nodal_PoissonRHS, P_old, Nodal_charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + ComputePoissonRHS(Nodal_PoissonRHS, P_old, Nodal_charge_den, MaterialMask, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta, geom); //Compute b - A x_H MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); @@ -353,6 +363,10 @@ void main_main (c_FerroX& rFerroX) //Poisson Solve pMLMG->solve({&Nodal_PoissonPhi}, {&Nodal_PoissonRHS}, 1.e-10, -1); + + //Fill inhomogenous Dirichlet BC in boundary nodes after the solve + SetPhiBC_z_after_solve(Nodal_PoissonPhi, n_cell); + Nodal_PoissonPhi.FillBoundary(geom.periodicity()); // Calculate rho from Phi in SC region @@ -387,6 +401,7 @@ void main_main (c_FerroX& rFerroX) amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); amrex::average_node_to_cellcenter(PoissonRHS, 0, Nodal_PoissonRHS, 0, 1); + amrex::average_node_to_cellcenter(angle_beta, 0, Nodal_angle_beta, 0, 1); // Write a plotfile of the initial data if plot_int > 0 if (plot_int > 0) { @@ -456,8 +471,17 @@ void main_main (c_FerroX& rFerroX) // iterate to compute Phi^{n+1,*} while(err > tol){ - // Compute RHS of Poisson equation - ComputePoissonRHS(Nodal_PoissonRHS, P_new_pre, Nodal_charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + //Compute x_H + SetPhiBC_z(Nodal_PoissonPhi_BC, n_cell); + + //Compute A x_H + pMLMG->apply ({&APoissonPhi_BC}, {&Nodal_PoissonPhi_BC}); + + //Compute b + ComputePoissonRHS(Nodal_PoissonRHS, P_new_pre, Nodal_charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + + //Compute b - A x_H + MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); //dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_new_pre, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); @@ -474,6 +498,9 @@ void main_main (c_FerroX& rFerroX) //Poisson Solve pMLMG->solve({&Nodal_PoissonPhi}, {&Nodal_PoissonRHS}, 1.e-10, -1); + //Fill inhomogenous Dirichlet BC in boundary nodes after the solve + SetPhiBC_z_after_solve(Nodal_PoissonPhi, n_cell); + Nodal_PoissonPhi.FillBoundary(geom.periodicity()); // Calculate rho from Phi in SC region @@ -528,8 +555,17 @@ void main_main (c_FerroX& rFerroX) // iterate to compute Phi^{n+1} while(err > tol){ - // Compute RHS of Poisson equation - ComputePoissonRHS(Nodal_PoissonRHS, P_new, Nodal_charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + //Compute x_H + SetPhiBC_z(Nodal_PoissonPhi_BC, n_cell); + + //Compute A x_H + pMLMG->apply ({&APoissonPhi_BC}, {&Nodal_PoissonPhi_BC}); + + //Compute b + ComputePoissonRHS(Nodal_PoissonRHS, P_new, Nodal_charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + + //Compute b - A x_H + MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); //dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_new, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); @@ -546,6 +582,10 @@ void main_main (c_FerroX& rFerroX) //Poisson Solve pMLMG->solve({&Nodal_PoissonPhi}, {&Nodal_PoissonRHS}, 1.e-10, -1); + + //Fill inhomogenous Dirichlet BC in boundary nodes after the solve + SetPhiBC_z_after_solve(Nodal_PoissonPhi, n_cell); + Nodal_PoissonPhi.FillBoundary(geom.periodicity()); // Calculate rho from Phi in SC region @@ -673,7 +713,7 @@ void main_main (c_FerroX& rFerroX) amrex::Print() << "step = " << step << ", Phi_Bc_hi = " << Phi_Bc_hi << std::endl; // Set Dirichlet BC for Phi in z - SetPhiBC_z(Nodal_PoissonPhi, n_cell); + //SetPhiBC_z(Nodal_PoissonPhi, n_cell); // set Dirichlet BC by reading in the ghost cell values //#ifdef AMREX_USE_EB @@ -688,8 +728,17 @@ void main_main (c_FerroX& rFerroX) // iterate to compute Phi^{n+1} with new Dirichlet value while(err > tol){ - // Compute RHS of Poisson equation - ComputePoissonRHS(Nodal_PoissonRHS, P_old, Nodal_charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + //Compute x_H + SetPhiBC_z(Nodal_PoissonPhi_BC, n_cell); + + //Compute A x_H + pMLMG->apply ({&APoissonPhi_BC}, {&Nodal_PoissonPhi_BC}); + + //Compute b + ComputePoissonRHS(Nodal_PoissonRHS, P_old, Nodal_charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + + //Compute b - A x_H + MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); //dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); @@ -706,6 +755,10 @@ void main_main (c_FerroX& rFerroX) //Poisson Solve pMLMG->solve({&Nodal_PoissonPhi}, {&Nodal_PoissonRHS}, 1.e-10, -1); + + //Fill inhomogenous Dirichlet BC in boundary nodes after the solve + SetPhiBC_z_after_solve(Nodal_PoissonPhi, n_cell); + Nodal_PoissonPhi.FillBoundary(geom.periodicity()); // Calculate rho from Phi in SC region From d86e22b54048fda1584285e055fce4ddcbf2a2b7 Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Fri, 21 Jul 2023 10:55:00 -0700 Subject: [PATCH 07/22] nodal angle paprser. There are problems --- Source/Solver/ElectrostaticSolver.cpp | 6 +- Source/Solver/Initialization.cpp | 109 ++++---------------------- Source/main.cpp | 22 +++--- 3 files changed, 30 insertions(+), 107 deletions(-) diff --git a/Source/Solver/ElectrostaticSolver.cpp b/Source/Solver/ElectrostaticSolver.cpp index 1eaee89..bf70e66 100644 --- a/Source/Solver/ElectrostaticSolver.cpp +++ b/Source/Solver/ElectrostaticSolver.cpp @@ -33,9 +33,9 @@ void ComputePoissonRHS(MultiFab& PoissonRHS, //Convert Euler angles from degrees to radians amrex::Real Pi = 3.14159265358979323846; -// amrex::Real alpha_rad = 0.*Pi/180.;//Pi/180.*angle_alpha_arr(i,j,k); -// amrex::Real beta_rad = 45.*Pi/180.;//Pi/180.*angle_beta_arr(i,j,k); -// amrex::Real theta_rad = 0.*Pi/180.;//Pi/180.*angle_theta_arr(i,j,k); + // amrex::Real alpha_rad = 0.*Pi/180.;//Pi/180.*angle_alpha_arr(i,j,k); + // amrex::Real beta_rad = 45.*Pi/180.;//Pi/180.*angle_beta_arr(i,j,k); + // amrex::Real theta_rad = 0.*Pi/180.;//Pi/180.*angle_theta_arr(i,j,k); amrex::Real alpha_rad = Pi/180.*angle_alpha_arr(i,j,k); amrex::Real beta_rad = Pi/180.*angle_beta_arr(i,j,k); amrex::Real theta_rad = Pi/180.*angle_theta_arr(i,j,k); diff --git a/Source/Solver/Initialization.cpp b/Source/Solver/Initialization.cpp index 3217be0..6a32cc4 100644 --- a/Source/Solver/Initialization.cpp +++ b/Source/Solver/Initialization.cpp @@ -59,8 +59,8 @@ void InitializePandRho(Array &P_old, rngs[i] = amrex::Random(); // uniform [0,1] option } - // loop over boxes - for (MFIter mfi(rho); mfi.isValid(); ++mfi) + // loop over cc boxes for P + for (MFIter mfi(P_old[0]); mfi.isValid(); ++mfi) { const Box& bx = mfi.validbox(); @@ -116,6 +116,18 @@ void InitializePandRho(Array &P_old, pOld_p(i,j,k) = 0.0; pOld_q(i,j,k) = 0.0; }); + } + + for (int i = 0; i < 3; i++){ + // fill periodic ghost cells + P_old[i].FillBoundary(geom.periodicity()); + } + + // loop over nodal boxes for rho + for (MFIter mfi(rho); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.validbox(); + // Calculate charge density from Phi, Nc, Nv, Ec, and Ev MultiFab acceptor_den(rho.boxArray(), rho.DistributionMap(), 1, 0); @@ -127,6 +139,7 @@ void InitializePandRho(Array &P_old, const Array4& acceptor_den_arr = acceptor_den.array(mfi); const Array4& donor_den_arr = donor_den.array(mfi); + const Array4& mask = MaterialMask.array(mfi); amrex::ParallelFor( bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) { @@ -171,10 +184,6 @@ void InitializePandRho(Array &P_old, } }); } - for (int i = 0; i < 3; i++){ - // fill periodic ghost cells - P_old[i].FillBoundary(geom.periodicity()); - } } @@ -396,91 +405,3 @@ void Initialize_Euler_angles(c_FerroX& rFerroX, const Geometry& geom, MultiFab& angle_theta.FillBoundary(geom.periodicity()); } -// initialization of Euler angles on the nodes -void Initialize_nodal_Euler_angles(c_FerroX& rFerroX, const Geometry& geom, MultiFab& Nodal_angle_alpha, MultiFab& Nodal_angle_beta, MultiFab& Nodal_angle_theta) -{ - auto& rGprop = rFerroX.get_GeometryProperties(); - Box const& domain = rGprop.geom.Domain(); - - const auto dx = rGprop.geom.CellSizeArray(); - const auto& real_box = rGprop.geom.ProbDomain(); - const auto iv_alpha = Nodal_angle_alpha.ixType().toIntVect(); - const auto iv_beta = Nodal_angle_beta.ixType().toIntVect(); - const auto iv_theta = Nodal_angle_theta.ixType().toIntVect(); - - for (MFIter mfi(Nodal_angle_alpha); mfi.isValid(); ++mfi) - { - const auto& alpha_arr = Nodal_angle_alpha.array(mfi); - const auto& beta_arr = Nodal_angle_beta.array(mfi); - const auto& theta_arr = Nodal_angle_theta.array(mfi); - const auto& bx = mfi.validbox(); - - std::string alpha_s; - std::unique_ptr alpha_parser; - std::string m_str_alpha_function; - - std::string beta_s; - std::unique_ptr beta_parser; - std::string m_str_beta_function; - - std::string theta_s; - std::unique_ptr theta_parser; - std::string m_str_theta_function; - - ParmParse pp_alpha("angle_alpha"); - - - if (pp_alpha.query("alpha_function(x,y,z)", m_str_alpha_function) ) { - alpha_s = "parse_alpha_function"; - } - - if (alpha_s == "parse_alpha_function") { - Store_parserString(pp_alpha, "alpha_function(x,y,z)", m_str_alpha_function); - alpha_parser = std::make_unique( - makeParser(m_str_alpha_function,{"x","y","z"})); - } - - ParmParse pp_beta("angle_beta"); - - - if (pp_beta.query("beta_function(x,y,z)", m_str_beta_function) ) { - beta_s = "parse_beta_function"; - } - - if (beta_s == "parse_beta_function") { - Store_parserString(pp_beta, "beta_function(x,y,z)", m_str_beta_function); - beta_parser = std::make_unique( - makeParser(m_str_beta_function,{"x","y","z"})); - } - - ParmParse pp_theta("angle_theta"); - - - if (pp_theta.query("theta_function(x,y,z)", m_str_theta_function) ) { - theta_s = "parse_theta_function"; - } - - if (theta_s == "parse_theta_function") { - Store_parserString(pp_theta, "theta_function(x,y,z)", m_str_theta_function); - theta_parser = std::make_unique( - makeParser(m_str_theta_function,{"x","y","z"})); - } - - const auto& macro_parser_alpha = alpha_parser->compile<3>(); - const auto& macro_parser_beta = beta_parser->compile<3>(); - const auto& macro_parser_theta = theta_parser->compile<3>(); - - amrex::ParallelFor(bx, - [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - eXstatic_MFab_Util::ConvertParserIntoMultiFab_3vars(i,j,k,dx,real_box,iv_alpha,macro_parser_alpha,alpha_arr); - eXstatic_MFab_Util::ConvertParserIntoMultiFab_3vars(i,j,k,dx,real_box,iv_beta, macro_parser_beta, beta_arr ); - eXstatic_MFab_Util::ConvertParserIntoMultiFab_3vars(i,j,k,dx,real_box,iv_theta,macro_parser_theta,theta_arr); - }); - - } - Nodal_angle_alpha.FillBoundary(geom.periodicity()); - Nodal_angle_beta.FillBoundary(geom.periodicity()); - Nodal_angle_theta.FillBoundary(geom.periodicity()); -} - diff --git a/Source/main.cpp b/Source/main.cpp index 07bcfac..02cffd6 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -154,12 +154,8 @@ void main_main (c_FerroX& rFerroX) //InitializeMaterialMask(rFerroX, geom, MaterialMask); if(Coordinate_Transformation == 1){ Initialize_tphase_Mask(rFerroX, geom, tphaseMask); - //Initialize_Euler_angles(rFerroX, geom, angle_alpha, angle_beta, angle_theta); - //average_cc_to_nodes(Nodal_angle_alpha, angle_alpha, geom); - //average_cc_to_nodes(Nodal_angle_beta, angle_beta, geom); - //average_cc_to_nodes(Nodal_angle_theta, angle_theta, geom); - - Initialize_nodal_Euler_angles(rFerroX, geom, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta); + Initialize_Euler_angles(rFerroX, geom, angle_alpha, angle_beta, angle_theta); //cell-centered + Initialize_Euler_angles(rFerroX, geom, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta); //nodal } else { tphaseMask.setVal(0.); } @@ -401,7 +397,10 @@ void main_main (c_FerroX& rFerroX) amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); amrex::average_node_to_cellcenter(PoissonRHS, 0, Nodal_PoissonRHS, 0, 1); - amrex::average_node_to_cellcenter(angle_beta, 0, Nodal_angle_beta, 0, 1); + amrex::average_node_to_cellcenter(hole_den, 0, Nodal_hole_den, 0, 1); + amrex::average_node_to_cellcenter(e_den, 0, Nodal_e_den, 0, 1); + amrex::average_node_to_cellcenter(charge_den, 0, Nodal_charge_den, 0, 1); + //amrex::average_node_to_cellcenter(angle_beta, 0, Nodal_angle_beta, 0, 1); // Write a plotfile of the initial data if plot_int > 0 if (plot_int > 0) { @@ -478,7 +477,7 @@ void main_main (c_FerroX& rFerroX) pMLMG->apply ({&APoissonPhi_BC}, {&Nodal_PoissonPhi_BC}); //Compute b - ComputePoissonRHS(Nodal_PoissonRHS, P_new_pre, Nodal_charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + ComputePoissonRHS(Nodal_PoissonRHS, P_new_pre, Nodal_charge_den, MaterialMask, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta, geom); //Compute b - A x_H MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); @@ -562,7 +561,7 @@ void main_main (c_FerroX& rFerroX) pMLMG->apply ({&APoissonPhi_BC}, {&Nodal_PoissonPhi_BC}); //Compute b - ComputePoissonRHS(Nodal_PoissonRHS, P_new, Nodal_charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + ComputePoissonRHS(Nodal_PoissonRHS, P_new, Nodal_charge_den, MaterialMask, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta, geom); //Compute b - A x_H MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); @@ -673,6 +672,9 @@ void main_main (c_FerroX& rFerroX) amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); amrex::average_node_to_cellcenter(PoissonRHS, 0, Nodal_PoissonRHS, 0, 1); + amrex::average_node_to_cellcenter(hole_den, 0, Nodal_hole_den, 0, 1); + amrex::average_node_to_cellcenter(e_den, 0, Nodal_e_den, 0, 1); + amrex::average_node_to_cellcenter(charge_den, 0, Nodal_charge_den, 0, 1); // Write a plotfile of the current data (plot_int was defined in the inputs file) if (plot_int > 0 && (step%plot_int == 0 || step == steady_state_step)) { @@ -735,7 +737,7 @@ void main_main (c_FerroX& rFerroX) pMLMG->apply ({&APoissonPhi_BC}, {&Nodal_PoissonPhi_BC}); //Compute b - ComputePoissonRHS(Nodal_PoissonRHS, P_old, Nodal_charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + ComputePoissonRHS(Nodal_PoissonRHS, P_old, Nodal_charge_den, MaterialMask, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta, geom); //Compute b - A x_H MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); From 1e18b097f002e0aad924238f455f3a25af50e2b7 Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Thu, 24 Aug 2023 16:14:40 -0700 Subject: [PATCH 08/22] implement fixes to rho etc that was recently made in dev --- Source/FerroX.cpp | 8 ++ Source/FerroX_namespace.H | 2 + Source/Solver/ChargeDensity.cpp | 66 ++++++++------- Source/Solver/ElectrostaticSolver.cpp | 9 +- Source/Solver/Initialization.H | 5 ++ Source/Solver/Initialization.cpp | 116 +++++++++++++++++++------- Source/main.cpp | 51 ++++++++--- 7 files changed, 180 insertions(+), 77 deletions(-) diff --git a/Source/FerroX.cpp b/Source/FerroX.cpp index 861ba14..57618a4 100644 --- a/Source/FerroX.cpp +++ b/Source/FerroX.cpp @@ -206,6 +206,8 @@ AMREX_GPU_MANAGED amrex::Real FerroX::kb; AMREX_GPU_MANAGED amrex::Real FerroX::T; AMREX_GPU_MANAGED amrex::Real FerroX::acceptor_doping; AMREX_GPU_MANAGED amrex::Real FerroX::donor_doping; +AMREX_GPU_MANAGED amrex::Real FerroX::intrinsic_carrier_concentration; +AMREX_GPU_MANAGED int FerroX::use_Fermi_Dirac; // P and Phi Bc AMREX_GPU_MANAGED amrex::Real FerroX::lambda; @@ -440,6 +442,12 @@ void InitializeFerroXNamespace(const amrex::GpuArray= 2.0) { - //Maxwell-Boltzmann -// hole_den_arr(i,j,k) = Nv*exp(-(q*phi(i,j,k) - Ev*1.602e-19)/(kb*T)); -// e_den_arr(i,j,k) = Nc*exp(-(Ec*1.602e-19 - q*phi(i,j,k))/(kb*T)); -// charge_den_arr(i,j,k) = q*(hole_den_arr(i,j,k) - e_den_arr(i,j,k)); - - //Fermi-Dirac - Real eta_n = q*(phi(i,j,k) - Ec)/(kb*T); - Real nu_n = std::pow(eta_n, 4.0) + 50.0 + 33.6 * eta_n * (1 - 0.68 * exp(-0.17 * std::pow((eta_n + 1), 2))); - Real xi_n = 3.0 * sqrt(3.14)/(4.0 * std::pow(nu_n, 3/8)); - Real FD_half_n = std::pow(exp(-eta_n) + xi_n, -1.0); - - e_den_arr(i,j,k) = 2.0/sqrt(3.14)*Nc*FD_half_n; - - Real eta_p = q*(Ev - phi(i,j,k))/(kb*T); - Real nu_p = std::pow(eta_p, 4.0) + 50.0 + 33.6 * eta_p * (1 - 0.68 * exp(-0.17 * std::pow((eta_p + 1), 2))); - Real xi_p = 3.0 * sqrt(3.14)/(4.0 * std::pow(nu_p, 3/8)); - Real FD_half_p = std::pow(exp(-eta_p) + xi_p, -1.0); - - hole_den_arr(i,j,k) = 2.0/sqrt(3.14)*Nv*FD_half_p; - - //If in channel, set acceptor doping, else (Source/Drain) set donor doping - if (mask(i,j,k) == 3.0) { - acceptor_den_arr(i,j,k) = acceptor_doping; - donor_den_arr(i,j,k) = 0.0; - } else { // Source / Drain - acceptor_den_arr(i,j,k) = 0.0; - donor_den_arr(i,j,k) = donor_doping; - } - - charge_den_arr(i,j,k) = q*(hole_den_arr(i,j,k) - e_den_arr(i,j,k) - acceptor_den_arr(i,j,k) + donor_den_arr(i,j,k)); + if(use_Fermi_Dirac == 1){ + //Fermi-Dirac + Real eta_n = q*(phi(i,j,k) - Ec)/(kb*T); + Real nu_n = std::pow(eta_n, 4.0) + 50.0 + 33.6 * eta_n * (1 - 0.68 * exp(-0.17 * std::pow((eta_n + 1), 2))); + Real xi_n = 3.0 * sqrt(3.14)/(4.0 * std::pow(nu_n, 3/8)); + Real FD_half_n = std::pow(exp(-eta_n) + xi_n, -1.0); + + e_den_arr(i,j,k) = 2.0/sqrt(3.14)*Nc*FD_half_n; + + Real eta_p = q*(Ev - phi(i,j,k))/(kb*T); + Real nu_p = std::pow(eta_p, 4.0) + 50.0 + 33.6 * eta_p * (1 - 0.68 * exp(-0.17 * std::pow((eta_p + 1), 2))); + Real xi_p = 3.0 * sqrt(3.14)/(4.0 * std::pow(nu_p, 3/8)); + Real FD_half_p = std::pow(exp(-eta_p) + xi_p, -1.0); + + hole_den_arr(i,j,k) = 2.0/sqrt(3.14)*Nv*FD_half_p; + } else { + //Maxwell-Boltzmann + acceptor_den_arr(i,j,k) = acceptor_doping; + donor_den_arr(i,j,k) = 0.0; + Real p_0 = acceptor_doping; + Real n_0 = intrinsic_carrier_concentration*intrinsic_carrier_concentration/p_0; + hole_den_arr(i,j,k) = p_0*exp(-(q*phi(i,j,k))/(kb*T)); + e_den_arr(i,j,k) = n_0*exp(q*phi(i,j,k)/(kb*T)); + } + + ////If in channel, set acceptor doping, else (Source/Drain) set donor doping + //if (mask(i,j,k) == 3.0) { + // acceptor_den_arr(i,j,k) = acceptor_doping; + // donor_den_arr(i,j,k) = 0.0; + //} else { // Source / Drain + // acceptor_den_arr(i,j,k) = 0.0; + // donor_den_arr(i,j,k) = donor_doping; + //} + + charge_den_arr(i,j,k) = q*(hole_den_arr(i,j,k) - e_den_arr(i,j,k) - acceptor_den_arr(i,j,k) + donor_den_arr(i,j,k)); } else { @@ -69,4 +74,3 @@ void ComputeRho(MultiFab& PoissonPhi, }); } } - diff --git a/Source/Solver/ElectrostaticSolver.cpp b/Source/Solver/ElectrostaticSolver.cpp index bf70e66..9ae0284 100644 --- a/Source/Solver/ElectrostaticSolver.cpp +++ b/Source/Solver/ElectrostaticSolver.cpp @@ -79,12 +79,10 @@ void ComputePoissonRHS(MultiFab& PoissonRHS, - (R_31*NodalDPDx(pOld_r, mask, i, j, k, dx) + R_32*NodalDPDy(pOld_r, mask, i, j, k, dx) + R_33*NodalDPDz(pOld_r, mask, i, j, k, dx)); RHS(i,j,k) *= -1.; -// RHS(i,j,k) = -0.; } }); } - } void dF_dPhi(MultiFab& alpha_cc, @@ -109,7 +107,7 @@ void dF_dPhi(MultiFab& alpha_cc, PoissonPhi_plus_delta.plus(delta, 0, 1, 0); // Calculate rho from Phi in SC region - ComputeRho(PoissonPhi, rho, e_den, p_den, MaterialMask); + ComputeRho(PoissonPhi_plus_delta, rho, e_den, p_den, MaterialMask); //Compute RHS of Poisson equation ComputePoissonRHS(PoissonRHS_phi_plus_delta, P_old, rho, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); @@ -208,6 +206,11 @@ void ComputeEfromPhi(MultiFab& PoissonPhi, }); } + for (int i = 0; i < 3; i++){ + // fill periodic ghost cells + E[i].FillBoundary(geom.periodicity()); + } + } void InitializePermittivity(std::array,2>& LinOpBCType_2d, diff --git a/Source/Solver/Initialization.H b/Source/Solver/Initialization.H index ab5813e..59e7ab9 100644 --- a/Source/Solver/Initialization.H +++ b/Source/Solver/Initialization.H @@ -24,6 +24,11 @@ void InitializeMaterialMask(MultiFab& MaterialMask, const amrex::GpuArray& prob_lo, const amrex::GpuArray& prob_hi); +void InitializeMaterialMask_Nodal(MultiFab& MaterialMask, + const Geometry& geom, + const amrex::GpuArray& prob_lo, + const amrex::GpuArray& prob_hi); + void InitializeMaterialMask(c_FerroX& rFerroX, const Geometry& geom, MultiFab& MaterialMask); void Initialize_tphase_Mask(c_FerroX& rFerroX, const Geometry& geom, MultiFab& tphaseMask); void Initialize_Euler_angles(c_FerroX& rFerroX, const Geometry& geom, MultiFab& angle_alpha, MultiFab& angle_beta, MultiFab& angle_theta); diff --git a/Source/Solver/Initialization.cpp b/Source/Solver/Initialization.cpp index 6a32cc4..a663f3b 100644 --- a/Source/Solver/Initialization.cpp +++ b/Source/Solver/Initialization.cpp @@ -147,46 +147,97 @@ void InitializePandRho(Array &P_old, //SC region if (mask(i,j,k) >= 2.0) { - Real Phi = 0.5*(Ec + Ev); //eV -// hole_den_arr(i,j,k) = Nv*exp(-(Phi - Ev)*1.602e-19/(kb*T)); -// e_den_arr(i,j,k) = Nc*exp(-(Ec - Phi)*1.602e-19/(kb*T)); -// charge_den_arr(i,j,k) = q*(hole_den_arr(i,j,k) - e_den_arr(i,j,k)); - - //Approximate FD integral - Real eta_n = q*(Phi - Ec)/(kb*T); - Real nu_n = std::pow(eta_n, 4.0) + 50.0 + 33.6 * eta_n * (1 - 0.68 * exp(-0.17 * std::pow((eta_n + 1), 2.0))); - Real xi_n = 3.0 * sqrt(3.14)/(4.0 * std::pow(nu_n, 3/8)); - Real FD_half_n = std::pow(exp(-eta_n) + xi_n, -1.0); - - e_den_arr(i,j,k) = 2.0/sqrt(3.14)*Nc*FD_half_n; - - Real eta_p = q*(Ev - Phi)/(kb*T); - Real nu_p = std::pow(eta_p, 4.0) + 50.0 + 33.6 * eta_p * (1 - 0.68 * exp(-0.17 * std::pow((eta_p + 1), 2.0))); - Real xi_p = 3.0 * sqrt(3.14)/(4.0 * std::pow(nu_p, 3/8)); - Real FD_half_p = std::pow(exp(-eta_p) + xi_p, -1.0); - - hole_den_arr(i,j,k) = 2.0/sqrt(3.14)*Nv*FD_half_p; - - //If in channel, set acceptor doping, else (Source/Drain) set donor doping - if (mask(i,j,k) == 3.0) { - acceptor_den_arr(i,j,k) = acceptor_doping; - donor_den_arr(i,j,k) = 0.0; - } else { // Source / Drain - acceptor_den_arr(i,j,k) = 0.0; - donor_den_arr(i,j,k) = donor_doping; - } - charge_den_arr(i,j,k) = q*(hole_den_arr(i,j,k) - e_den_arr(i,j,k) - acceptor_den_arr(i,j,k) + donor_den_arr(i,j,k)); + if(use_Fermi_Dirac == 1){ + + //Approximate FD integral + Real Phi = 0.5*(Ec + Ev); //eV + Real eta_n = q*(Phi - Ec)/(kb*T); + Real nu_n = std::pow(eta_n, 4.0) + 50.0 + 33.6 * eta_n * (1 - 0.68 * exp(-0.17 * std::pow((eta_n + 1), 2.0))); + Real xi_n = 3.0 * sqrt(3.14)/(4.0 * std::pow(nu_n, 3/8)); + Real FD_half_n = std::pow(exp(-eta_n) + xi_n, -1.0); + + e_den_arr(i,j,k) = 2.0/sqrt(3.14)*Nc*FD_half_n; + + Real eta_p = q*(Ev - Phi)/(kb*T); + Real nu_p = std::pow(eta_p, 4.0) + 50.0 + 33.6 * eta_p * (1 - 0.68 * exp(-0.17 * std::pow((eta_p + 1), 2.0))); + Real xi_p = 3.0 * sqrt(3.14)/(4.0 * std::pow(nu_p, 3/8)); + Real FD_half_p = std::pow(exp(-eta_p) + xi_p, -1.0); + + hole_den_arr(i,j,k) = 2.0/sqrt(3.14)*Nv*FD_half_p; + + } else { + + acceptor_den_arr(i,j,k) = acceptor_doping; + donor_den_arr(i,j,k) = 0.0; + hole_den_arr(i,j,k) = acceptor_doping; + e_den_arr(i,j,k) = intrinsic_carrier_concentration*intrinsic_carrier_concentration/acceptor_doping; + } + charge_den_arr(i,j,k) = q*(hole_den_arr(i,j,k) - e_den_arr(i,j,k) - acceptor_den_arr(i,j,k) + donor_den_arr(i,j,k)); } else { + charge_den_arr(i,j,k) = 0.; + } - charge_den_arr(i,j,k) = 0.0; +// //If in channel, set acceptor doping, else (Source/Drain) set donor doping +// if (mask(i,j,k) == 3.0) { +// acceptor_den_arr(i,j,k) = acceptor_doping; +// donor_den_arr(i,j,k) = 0.0; +// } else { // Source / Drain +// acceptor_den_arr(i,j,k) = 0.0; +// donor_den_arr(i,j,k) = donor_doping; +// } - } }); } + rho.FillBoundary(geom.periodicity()); + e_den.FillBoundary(geom.periodicity()); + p_den.FillBoundary(geom.periodicity()); } +// create a mask filled with integers to idetify different material types +void InitializeMaterialMask_Nodal(MultiFab& MaterialMask, + const Geometry& geom, + const amrex::GpuArray& prob_lo, + const amrex::GpuArray& prob_hi) +{ + // loop over boxes + for (MFIter mfi(MaterialMask); mfi.isValid(); ++mfi) + { + //const Box& bx = mfi.validbox(); + const Box& bx = mfi.growntilebox(MaterialMask.nGrow()); + + // extract dx from the geometry object + GpuArray dx = geom.CellSizeArray(); + + const Array4& mask = MaterialMask.array(mfi); + + + amrex::ParallelFor( bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + Real x = prob_lo[0] + i* dx[0]; + Real y = prob_lo[1] + j* dx[1]; + Real z = prob_lo[2] + k* dx[2]; + + //FE:0, DE:1, Source/Drain:2, Channel:3 + if (x <= FE_hi[0] && x >= FE_lo[0] && y <= FE_hi[1] && y >= FE_lo[1] && z <= FE_hi[2] && z >= FE_lo[2]) { + mask(i,j,k) = 0.; + } else if (x <= DE_hi[0] && x >= DE_lo[0] && y <= DE_hi[1] && y >= DE_lo[1] && z <= DE_hi[2] && z >= DE_lo[2]) { + mask(i,j,k) = 1.; + } else if (x <= SC_hi[0] && x >= SC_lo[0] && y <= SC_hi[1] && y >= SC_lo[1] && z <= SC_hi[2] && z >= SC_lo[2]) { + mask(i,j,k) = 2.; + if (x <= Channel_hi[0] && x >= Channel_lo[0] && y <= Channel_hi[1] && y >= Channel_lo[1] && z <= Channel_hi[2] && z >= Channel_lo[2]){ + mask(i,j,k) = 3.; + } + } else { + mask(i,j,k) = 1.; //spacer is DE + } + }); + } + MaterialMask.FillBoundary(geom.periodicity()); +} + + // create a mask filled with integers to idetify different material types void InitializeMaterialMask(MultiFab& MaterialMask, const Geometry& geom, @@ -196,7 +247,8 @@ void InitializeMaterialMask(MultiFab& MaterialMask, // loop over boxes for (MFIter mfi(MaterialMask); mfi.isValid(); ++mfi) { - const Box& bx = mfi.validbox(); + //const Box& bx = mfi.validbox(); + const Box& bx = mfi.growntilebox(MaterialMask.nGrow()); // extract dx from the geometry object GpuArray dx = geom.CellSizeArray(); diff --git a/Source/main.cpp b/Source/main.cpp index 02cffd6..e7a836f 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -145,11 +145,40 @@ void main_main (c_FerroX& rFerroX) MultiFab angle_theta(ba, dm, 1, 0); //Nodal angles + MultiFab Nodal_MaterialMask(ba, dm, 1, 1); MultiFab Nodal_angle_alpha(nba, dm, 1, 0); MultiFab Nodal_angle_beta(nba, dm, 1, 0); MultiFab Nodal_angle_theta(nba, dm, 1, 0); + for (int dir = 0; dir < AMREX_SPACEDIM; dir++) + { + P_old[dir].setVal(0.); + P_new[dir].setVal(0.); + P_new_pre[dir].setVal(0.); + GL_rhs[dir].setVal(0.); + GL_rhs_pre[dir].setVal(0.); + GL_rhs_avg[dir].setVal(0.); + E[dir].setVal(0.); + } + + PoissonPhi.setVal(0.); + PoissonRHS.setVal(0.); + tphaseMask.setVal(0.); + angle_alpha.setVal(0.); + angle_beta.setVal(0.); + angle_theta.setVal(0.); + + Nodal_PoissonPhi.setVal(0.); + Nodal_PoissonRHS.setVal(0.); + Nodal_angle_alpha.setVal(0.); + Nodal_angle_beta.setVal(0.); + Nodal_angle_theta.setVal(0.); + Nodal_hole_den.setVal(0.); + Nodal_e_den.setVal(0.); + Nodal_charge_den.setVal(0.); + //Initialize material mask + InitializeMaterialMask_Nodal(Nodal_MaterialMask, geom, prob_lo, prob_hi); InitializeMaterialMask(MaterialMask, geom, prob_lo, prob_hi); //InitializeMaterialMask(rFerroX, geom, MaterialMask); if(Coordinate_Transformation == 1){ @@ -315,8 +344,8 @@ void main_main (c_FerroX& rFerroX) // INITIALIZE P in FE and rho in SC regions //InitializePandRho(P_old, Gamma, charge_den, e_den, hole_den, geom, prob_lo, prob_hi);//old - InitializePandRho(P_old, Gamma, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, MaterialMask, tphaseMask, n_cell, geom, prob_lo, prob_hi);//mask based - + InitializePandRho(P_old, Gamma, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, Nodal_MaterialMask, tphaseMask, n_cell, geom, prob_lo, prob_hi);//mask based + //Obtain self consisten Phi and rho Real tol = 1.e-5; Real err = 1.0; @@ -340,7 +369,7 @@ void main_main (c_FerroX& rFerroX) pMLMG->apply ({&APoissonPhi_BC}, {&Nodal_PoissonPhi_BC}); //Compute b - ComputePoissonRHS(Nodal_PoissonRHS, P_old, Nodal_charge_den, MaterialMask, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta, geom); + ComputePoissonRHS(Nodal_PoissonRHS, P_old, Nodal_charge_den, Nodal_MaterialMask, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta, geom); //Compute b - A x_H MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); @@ -368,7 +397,7 @@ void main_main (c_FerroX& rFerroX) // Calculate rho from Phi in SC region //ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); //Nodal - ComputeRho(Nodal_PoissonPhi, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, MaterialMask); + ComputeRho(Nodal_PoissonPhi, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, Nodal_MaterialMask); if (contains_SC == 0) { // no semiconductor region; set error to zero so the while loop terminates @@ -391,7 +420,7 @@ void main_main (c_FerroX& rFerroX) } amrex::Print() << "\n ========= Self-Consistent Initialization of P and Rho Done! ========== \n"<< iter << " iterations to obtain self consistent Phi with err = " << err << std::endl; - + // Calculate E from Phi ComputeEfromPhi(Nodal_PoissonPhi, E, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); @@ -477,7 +506,7 @@ void main_main (c_FerroX& rFerroX) pMLMG->apply ({&APoissonPhi_BC}, {&Nodal_PoissonPhi_BC}); //Compute b - ComputePoissonRHS(Nodal_PoissonRHS, P_new_pre, Nodal_charge_den, MaterialMask, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta, geom); + ComputePoissonRHS(Nodal_PoissonRHS, P_new_pre, Nodal_charge_den, Nodal_MaterialMask, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta, geom); //Compute b - A x_H MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); @@ -505,7 +534,7 @@ void main_main (c_FerroX& rFerroX) // Calculate rho from Phi in SC region //ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); //Nodal - ComputeRho(Nodal_PoissonPhi, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, MaterialMask); + ComputeRho(Nodal_PoissonPhi, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, Nodal_MaterialMask); if (contains_SC == 0) { // no semiconductor region; set error to zero so the while loop terminates @@ -561,7 +590,7 @@ void main_main (c_FerroX& rFerroX) pMLMG->apply ({&APoissonPhi_BC}, {&Nodal_PoissonPhi_BC}); //Compute b - ComputePoissonRHS(Nodal_PoissonRHS, P_new, Nodal_charge_den, MaterialMask, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta, geom); + ComputePoissonRHS(Nodal_PoissonRHS, P_new, Nodal_charge_den, Nodal_MaterialMask, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta, geom); //Compute b - A x_H MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); @@ -590,7 +619,7 @@ void main_main (c_FerroX& rFerroX) // Calculate rho from Phi in SC region //ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); //Nodal - ComputeRho(Nodal_PoissonPhi, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, MaterialMask); + ComputeRho(Nodal_PoissonPhi, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, Nodal_MaterialMask); if (contains_SC == 0) { // no semiconductor region; set error to zero so the while loop terminates @@ -737,7 +766,7 @@ void main_main (c_FerroX& rFerroX) pMLMG->apply ({&APoissonPhi_BC}, {&Nodal_PoissonPhi_BC}); //Compute b - ComputePoissonRHS(Nodal_PoissonRHS, P_old, Nodal_charge_den, MaterialMask, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta, geom); + ComputePoissonRHS(Nodal_PoissonRHS, P_old, Nodal_charge_den, Nodal_MaterialMask, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta, geom); //Compute b - A x_H MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); @@ -766,7 +795,7 @@ void main_main (c_FerroX& rFerroX) // Calculate rho from Phi in SC region //ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); //Nodal - ComputeRho(Nodal_PoissonPhi, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, MaterialMask); + ComputeRho(Nodal_PoissonPhi, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, Nodal_MaterialMask); if (contains_SC == 0) { // no semiconductor region; set error to zero so the while loop terminates From 613874a30c6f1e7441e92e7692a14552c96fc2f2 Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Thu, 24 Aug 2023 16:55:12 -0700 Subject: [PATCH 09/22] abort when coordinate transform is off but rotation matrix is not identity --- Source/Solver/ElectrostaticSolver.cpp | 25 + Source/Solver/TotalEnergyDensity.cpp | 190 +++---- Source/main_sweep.cpp | 650 ++++++++++++++++++++++++ Source/main_vsweep.cpp | 696 ++++++++++++++++++++++++++ 4 files changed, 1472 insertions(+), 89 deletions(-) create mode 100644 Source/main_sweep.cpp create mode 100644 Source/main_vsweep.cpp diff --git a/Source/Solver/ElectrostaticSolver.cpp b/Source/Solver/ElectrostaticSolver.cpp index 9ae0284..9506b36 100644 --- a/Source/Solver/ElectrostaticSolver.cpp +++ b/Source/Solver/ElectrostaticSolver.cpp @@ -64,6 +64,18 @@ void ComputePoissonRHS(MultiFab& PoissonRHS, R_33 = cos(alpha_rad)*cos(beta_rad); } + //when coordinate transformation is OFF, + //alpha = beta = theta = 0. + //Therefore, R_11 = R_22 = R_33 = 1, R_12 = R_13 = R_21 = R_23 = R_31 = R_32 = 0. + + if (Coordinate_Transformation != 1){ + if (R_11 != 1.0 || R_12 != 0.0 || R_13 != 0.0 || + R_21 != 0.0 || R_22 != 1.0 || R_23 != 0.0 || + R_31 != 0.0 || R_32 != 0.0 || R_33 != 1.0 ){ + amrex::Abort("Coordinate transformation is turned OFF, but rotation matrix is not an identity matrix!"); + } + } + if(mask(i,j,k) >= 2.0){ //SC region RHS(i,j,k) = charge_den_arr(i,j,k); @@ -198,6 +210,19 @@ void ComputeEfromPhi(MultiFab& PoissonPhi, R_33 = cos(alpha_rad)*cos(beta_rad); } + //when coordinate transformation is OFF, + //alpha = beta = theta = 0. + //Therefore, R_11 = R_22 = R_33 = 1, R_12 = R_13 = R_21 = R_23 = R_31 = R_32 = 0. + //So, Ep = Ex = -DFDx(phi), Eq = Ey = -DFDy(phi), Er = Ez = -DphiDz(phi) + + if (Coordinate_Transformation != 1){ + if (R_11 != 1.0 || R_12 != 0.0 || R_13 != 0.0 || + R_21 != 0.0 || R_22 != 1.0 || R_23 != 0.0 || + R_31 != 0.0 || R_32 != 0.0 || R_33 != 1.0 ){ + amrex::Abort("Coordinate transformation is turned OFF, but rotation matrix is not an identity matrix!"); + } + } + Ep_arr(i,j,k) = - (R_11*DFDx(phi, i, j, k, dx) + R_12*DFDy(phi, i, j, k, dx) + R_13*DphiDz(phi, z_hi, z_lo, i, j, k, dx, prob_lo, prob_hi)); Eq_arr(i,j,k) = - (R_21*DFDx(phi, i, j, k, dx) + R_22*DFDy(phi, i, j, k, dx) + R_23*DphiDz(phi, z_hi, z_lo, i, j, k, dx, prob_lo, prob_hi)); Er_arr(i,j,k) = - (R_31*DFDx(phi, i, j, k, dx) + R_32*DFDy(phi, i, j, k, dx) + R_33*DphiDz(phi, z_hi, z_lo, i, j, k, dx, prob_lo, prob_hi)); diff --git a/Source/Solver/TotalEnergyDensity.cpp b/Source/Solver/TotalEnergyDensity.cpp index a945f03..aed4696 100644 --- a/Source/Solver/TotalEnergyDensity.cpp +++ b/Source/Solver/TotalEnergyDensity.cpp @@ -73,95 +73,107 @@ void CalculateTDGL_RHS(Array &GL_rhs, R_33 = cos(alpha_rad)*cos(beta_rad); } - Real dFdPp_Landau = alpha*pOld_p(i,j,k) + beta*std::pow(pOld_p(i,j,k),3.) + FerroX::gamma*std::pow(pOld_p(i,j,k),5.) - + 2. * alpha_12 * pOld_p(i,j,k) * std::pow(pOld_q(i,j,k),2.) - + 2. * alpha_12 * pOld_p(i,j,k) * std::pow(pOld_r(i,j,k),2.) - + 4. * alpha_112 * std::pow(pOld_p(i,j,k),3.) * (std::pow(pOld_q(i,j,k),2.) + std::pow(pOld_r(i,j,k),2.)) - + 2. * alpha_112 * pOld_p(i,j,k) * std::pow(pOld_q(i,j,k),4.) - + 2. * alpha_112 * pOld_p(i,j,k) * std::pow(pOld_r(i,j,k),4.) - + 2. * alpha_123 * pOld_p(i,j,k) * std::pow(pOld_q(i,j,k),2.) * std::pow(pOld_r(i,j,k),2.); - - Real dFdPq_Landau = alpha*pOld_q(i,j,k) + beta*std::pow(pOld_q(i,j,k),3.) + FerroX::gamma*std::pow(pOld_q(i,j,k),5.) - + 2. * alpha_12 * pOld_q(i,j,k) * std::pow(pOld_p(i,j,k),2.) - + 2. * alpha_12 * pOld_q(i,j,k) * std::pow(pOld_r(i,j,k),2.) - + 4. * alpha_112 * std::pow(pOld_q(i,j,k),3.) * (std::pow(pOld_p(i,j,k),2.) + std::pow(pOld_r(i,j,k),2.)) - + 2. * alpha_112 * pOld_q(i,j,k) * std::pow(pOld_p(i,j,k),4.) - + 2. * alpha_112 * pOld_q(i,j,k) * std::pow(pOld_r(i,j,k),4.) - + 2. * alpha_123 * pOld_q(i,j,k) * std::pow(pOld_p(i,j,k),2.) * std::pow(pOld_r(i,j,k),2.); - - Real dFdPr_Landau = alpha*pOld_r(i,j,k) + beta*std::pow(pOld_r(i,j,k),3.) + FerroX::gamma*std::pow(pOld_r(i,j,k),5.) - + 2. * alpha_12 * pOld_r(i,j,k) * std::pow(pOld_p(i,j,k),2.) - + 2. * alpha_12 * pOld_r(i,j,k) * std::pow(pOld_q(i,j,k),2.) - + 4. * alpha_112 * std::pow(pOld_r(i,j,k),3.) * (std::pow(pOld_p(i,j,k),2.) + std::pow(pOld_q(i,j,k),2.)) - + 2. * alpha_112 * pOld_r(i,j,k) * std::pow(pOld_p(i,j,k),4.) - + 2. * alpha_112 * pOld_r(i,j,k) * std::pow(pOld_q(i,j,k),4.) - + 2. * alpha_123 * pOld_r(i,j,k) * std::pow(pOld_p(i,j,k),2.) * std::pow(pOld_q(i,j,k),2.); - - Real dFdPp_grad = - g11 * DoubleDPDx(pOld_p, mask, i, j, k, dx) - - (g44 + g44_p) * DoubleDPDy(pOld_p, mask, i, j, k, dx) - - (g44 + g44_p) * DoubleDPDz(pOld_p, mask, i, j, k, dx) - - (g12 + g44 - g44_p) * DoubleDPDxDy(pOld_q, mask, i, j, k, dx) // d2P/dxdy - - (g12 + g44 - g44_p) * DoubleDPDxDz(pOld_r, mask, i, j, k, dx); // d2P/dxdz - - Real dFdPq_grad = - g11 * DoubleDPDy(pOld_q, mask, i, j, k, dx) - - (g44 - g44_p) * DoubleDPDx(pOld_q, mask, i, j, k, dx) - - (g44 - g44_p) * DoubleDPDz(pOld_q, mask, i, j, k, dx) - - (g12 + g44 + g44_p) * DoubleDPDxDy(pOld_p, mask, i, j, k, dx) // d2P/dxdy - - (g12 + g44 - g44_p) * DoubleDPDyDz(pOld_r, mask, i, j, k, dx);// d2P/dydz - - //Switch g11 and g44 temporarily for multiphase simulations. This will be generalized later - Real dFdPr_grad = - g44 * ( R_31*R_31*DoubleDPDx(pOld_r, mask, i, j, k, dx) - +R_32*R_32*DoubleDPDy(pOld_r, mask, i, j, k, dx) - +R_33*R_33*DoubleDPDz(pOld_r, mask, i, j, k, dx) - +2.*R_31*R_32*DoubleDPDxDy(pOld_r, mask, i, j, k, dx) - +2.*R_32*R_33*DoubleDPDyDz(pOld_r, mask, i, j, k, dx) - +2.*R_33*R_31*DoubleDPDxDz(pOld_r, mask, i, j, k, dx)) - - - (g11 - g44_p) * ( R_11*R_11*DoubleDPDx(pOld_r, mask, i, j, k, dx) - +R_12*R_12*DoubleDPDy(pOld_r, mask, i, j, k, dx) - +R_13*R_13*DoubleDPDz(pOld_r, mask, i, j, k, dx) - +2.*R_11*R_12*DoubleDPDxDy(pOld_r, mask, i, j, k, dx) - +2.*R_12*R_13*DoubleDPDyDz(pOld_r, mask, i, j, k, dx) - +2.*R_13*R_11*DoubleDPDxDz(pOld_r, mask, i, j, k, dx)) - - - (g44 - g44_p) * ( R_21*R_21*DoubleDPDx(pOld_r, mask, i, j, k, dx) - +R_22*R_22*DoubleDPDy(pOld_r, mask, i, j, k, dx) - +R_23*R_23*DoubleDPDz(pOld_r, mask, i, j, k, dx) - +2.*R_21*R_22*DoubleDPDxDy(pOld_r, mask, i, j, k, dx) - +2.*R_22*R_23*DoubleDPDyDz(pOld_r, mask, i, j, k, dx) - +2.*R_23*R_21*DoubleDPDxDz(pOld_r, mask, i, j, k, dx)) - - - (g44 + g44_p + g12) * DoubleDPDyDz(pOld_q, mask, i, j, k, dx) // d2P/dydz - - (g44 + g44_p + g12) * DoubleDPDxDz(pOld_p, mask, i, j, k, dx); // d2P/dxdz - - GL_RHS_p(i,j,k) = -1.0 * Gam(i,j,k) * - ( dFdPp_Landau - + dFdPp_grad - - Ep(i,j,k) - //+ DFDx(phi, i, j, k, dx) - ); - - GL_RHS_q(i,j,k) = -1.0 * Gam(i,j,k) * - ( dFdPq_Landau - + dFdPq_grad - - Eq(i,j,k) - //+ DFDy(phi, i, j, k, dx) - ); - - GL_RHS_p(i,j,k) = 0.0; - GL_RHS_q(i,j,k) = 0.0; - GL_RHS_r(i,j,k) = -1.0 * Gam(i,j,k) * - ( dFdPr_Landau - + dFdPr_grad - - Er(i,j,k) - //+ DphiDz(phi, z_hi, z_lo, i, j, k, dx, prob_lo, prob_hi) - ); - - //set t_phase GL_RHS_r to zero so that it stays zero. It is initialized to zero in t-phase as well - //if(x <= t_phase_hi[0] && x >= t_phase_lo[0] && y <= t_phase_hi[1] && y >= t_phase_lo[1] && z <= t_phase_hi[2] && z >= t_phase_lo[2]){ - if(tphase(i,j,k) == 1.0){ - GL_RHS_r(i,j,k) = 0.0; - } + //when coordinate transformation is OFF, + //alpha = beta = theta = 0. + //Therefore, R_11 = R_22 = R_33 = 1, R_12 = R_13 = R_21 = R_23 = R_31 = R_32 = 0. + + if (Coordinate_Transformation != 1){ + if (R_11 != 1.0 || R_12 != 0.0 || R_13 != 0.0 || + R_21 != 0.0 || R_22 != 1.0 || R_23 != 0.0 || + R_31 != 0.0 || R_32 != 0.0 || R_33 != 1.0 ){ + amrex::Abort("Coordinate transformation is turned OFF, but rotation matrix is not an identity matrix!"); + } + } + + Real dFdPp_Landau = alpha*pOld_p(i,j,k) + beta*std::pow(pOld_p(i,j,k),3.) + FerroX::gamma*std::pow(pOld_p(i,j,k),5.) + + 2. * alpha_12 * pOld_p(i,j,k) * std::pow(pOld_q(i,j,k),2.) + + 2. * alpha_12 * pOld_p(i,j,k) * std::pow(pOld_r(i,j,k),2.) + + 4. * alpha_112 * std::pow(pOld_p(i,j,k),3.) * (std::pow(pOld_q(i,j,k),2.) + std::pow(pOld_r(i,j,k),2.)) + + 2. * alpha_112 * pOld_p(i,j,k) * std::pow(pOld_q(i,j,k),4.) + + 2. * alpha_112 * pOld_p(i,j,k) * std::pow(pOld_r(i,j,k),4.) + + 2. * alpha_123 * pOld_p(i,j,k) * std::pow(pOld_q(i,j,k),2.) * std::pow(pOld_r(i,j,k),2.); + + Real dFdPq_Landau = alpha*pOld_q(i,j,k) + beta*std::pow(pOld_q(i,j,k),3.) + FerroX::gamma*std::pow(pOld_q(i,j,k),5.) + + 2. * alpha_12 * pOld_q(i,j,k) * std::pow(pOld_p(i,j,k),2.) + + 2. * alpha_12 * pOld_q(i,j,k) * std::pow(pOld_r(i,j,k),2.) + + 4. * alpha_112 * std::pow(pOld_q(i,j,k),3.) * (std::pow(pOld_p(i,j,k),2.) + std::pow(pOld_r(i,j,k),2.)) + + 2. * alpha_112 * pOld_q(i,j,k) * std::pow(pOld_p(i,j,k),4.) + + 2. * alpha_112 * pOld_q(i,j,k) * std::pow(pOld_r(i,j,k),4.) + + 2. * alpha_123 * pOld_q(i,j,k) * std::pow(pOld_p(i,j,k),2.) * std::pow(pOld_r(i,j,k),2.); + + Real dFdPr_Landau = alpha*pOld_r(i,j,k) + beta*std::pow(pOld_r(i,j,k),3.) + FerroX::gamma*std::pow(pOld_r(i,j,k),5.) + + 2. * alpha_12 * pOld_r(i,j,k) * std::pow(pOld_p(i,j,k),2.) + + 2. * alpha_12 * pOld_r(i,j,k) * std::pow(pOld_q(i,j,k),2.) + + 4. * alpha_112 * std::pow(pOld_r(i,j,k),3.) * (std::pow(pOld_p(i,j,k),2.) + std::pow(pOld_q(i,j,k),2.)) + + 2. * alpha_112 * pOld_r(i,j,k) * std::pow(pOld_p(i,j,k),4.) + + 2. * alpha_112 * pOld_r(i,j,k) * std::pow(pOld_q(i,j,k),4.) + + 2. * alpha_123 * pOld_r(i,j,k) * std::pow(pOld_p(i,j,k),2.) * std::pow(pOld_q(i,j,k),2.); + + Real dFdPp_grad = - g11 * DoubleDPDx(pOld_p, mask, i, j, k, dx) + - (g44 + g44_p) * DoubleDPDy(pOld_p, mask, i, j, k, dx) + - (g44 + g44_p) * DoubleDPDz(pOld_p, mask, i, j, k, dx) + - (g12 + g44 - g44_p) * DoubleDPDxDy(pOld_q, mask, i, j, k, dx) // d2P/dxdy + - (g12 + g44 - g44_p) * DoubleDPDxDz(pOld_r, mask, i, j, k, dx); // d2P/dxdz + + Real dFdPq_grad = - g11 * DoubleDPDy(pOld_q, mask, i, j, k, dx) + - (g44 - g44_p) * DoubleDPDx(pOld_q, mask, i, j, k, dx) + - (g44 - g44_p) * DoubleDPDz(pOld_q, mask, i, j, k, dx) + - (g12 + g44 + g44_p) * DoubleDPDxDy(pOld_p, mask, i, j, k, dx) // d2P/dxdy + - (g12 + g44 - g44_p) * DoubleDPDyDz(pOld_r, mask, i, j, k, dx);// d2P/dydz + + //Switch g11 and g44 temporarily for multiphase simulations. This will be generalized later + Real dFdPr_grad = - g44 * ( R_31*R_31*DoubleDPDx(pOld_r, mask, i, j, k, dx) + +R_32*R_32*DoubleDPDy(pOld_r, mask, i, j, k, dx) + +R_33*R_33*DoubleDPDz(pOld_r, mask, i, j, k, dx) + +2.*R_31*R_32*DoubleDPDxDy(pOld_r, mask, i, j, k, dx) + +2.*R_32*R_33*DoubleDPDyDz(pOld_r, mask, i, j, k, dx) + +2.*R_33*R_31*DoubleDPDxDz(pOld_r, mask, i, j, k, dx)) + + - (g11 - g44_p) * ( R_11*R_11*DoubleDPDx(pOld_r, mask, i, j, k, dx) + +R_12*R_12*DoubleDPDy(pOld_r, mask, i, j, k, dx) + +R_13*R_13*DoubleDPDz(pOld_r, mask, i, j, k, dx) + +2.*R_11*R_12*DoubleDPDxDy(pOld_r, mask, i, j, k, dx) + +2.*R_12*R_13*DoubleDPDyDz(pOld_r, mask, i, j, k, dx) + +2.*R_13*R_11*DoubleDPDxDz(pOld_r, mask, i, j, k, dx)) + + - (g44 - g44_p) * ( R_21*R_21*DoubleDPDx(pOld_r, mask, i, j, k, dx) + +R_22*R_22*DoubleDPDy(pOld_r, mask, i, j, k, dx) + +R_23*R_23*DoubleDPDz(pOld_r, mask, i, j, k, dx) + +2.*R_21*R_22*DoubleDPDxDy(pOld_r, mask, i, j, k, dx) + +2.*R_22*R_23*DoubleDPDyDz(pOld_r, mask, i, j, k, dx) + +2.*R_23*R_21*DoubleDPDxDz(pOld_r, mask, i, j, k, dx)) + + - (g44 + g44_p + g12) * DoubleDPDyDz(pOld_q, mask, i, j, k, dx) // d2P/dydz + - (g44 + g44_p + g12) * DoubleDPDxDz(pOld_p, mask, i, j, k, dx); // d2P/dxdz + + GL_RHS_p(i,j,k) = -1.0 * Gam(i,j,k) * + ( dFdPp_Landau + + dFdPp_grad + - Ep(i,j,k) + //+ DFDx(phi, i, j, k, dx) + ); + + GL_RHS_q(i,j,k) = -1.0 * Gam(i,j,k) * + ( dFdPq_Landau + + dFdPq_grad + - Eq(i,j,k) + //+ DFDy(phi, i, j, k, dx) + ); + + GL_RHS_p(i,j,k) = 0.0; + GL_RHS_q(i,j,k) = 0.0; + GL_RHS_r(i,j,k) = -1.0 * Gam(i,j,k) * + ( dFdPr_Landau + + dFdPr_grad + - Er(i,j,k) + //+ DphiDz(phi, z_hi, z_lo, i, j, k, dx, prob_lo, prob_hi) + ); + + //set t_phase GL_RHS_r to zero so that it stays zero. It is initialized to zero in t-phase as well + //if(x <= t_phase_hi[0] && x >= t_phase_lo[0] && y <= t_phase_hi[1] && y >= t_phase_lo[1] && z <= t_phase_hi[2] && z >= t_phase_lo[2]){ + if(tphase(i,j,k) == 1.0){ + GL_RHS_r(i,j,k) = 0.0; + } }); } } diff --git a/Source/main_sweep.cpp b/Source/main_sweep.cpp new file mode 100644 index 0000000..5f73b1c --- /dev/null +++ b/Source/main_sweep.cpp @@ -0,0 +1,650 @@ + +#include +#include +#include +#ifdef AMREX_USE_EB +#include +#endif +#include +#include +#include +#include "FerroX.H" +#include "Solver/ElectrostaticSolver.H" +#include "Solver/Initialization.H" +#include "Solver/ChargeDensity.H" +#include "Solver/TotalEnergyDensity.H" +#include "Input/BoundaryConditions/BoundaryConditions.H" +#include "Input/GeometryProperties/GeometryProperties.H" +#include "Utils/SelectWarpXUtils/WarpXUtil.H" +#include "Utils/SelectWarpXUtils/WarpXProfilerWrapper.H" +#include "Utils/eXstaticUtils/eXstaticUtil.H" +#include "Utils/FerroXUtils/FerroXUtil.H" + + + + +using namespace amrex; + +using namespace FerroX; + +int main (int argc, char* argv[]) +{ + amrex::Initialize(argc,argv); + + { + c_FerroX pFerroX; + pFerroX.InitData(); + main_main(pFerroX); + } + amrex::Finalize(); + return 0; +} + +void main_main (c_FerroX& rFerroX) +{ + + Real total_step_strt_time = ParallelDescriptor::second(); + + auto& rGprop = rFerroX.get_GeometryProperties(); + auto& geom = rGprop.geom; + auto& ba = rGprop.ba; + auto& dm = rGprop.dm; + auto& is_periodic = rGprop.is_periodic; + auto& prob_lo = rGprop.prob_lo; + auto& prob_hi = rGprop.prob_hi; + auto& n_cell = rGprop.n_cell; + + // read in inputs file + InitializeFerroXNamespace(prob_lo, prob_hi); + + // Nghost = number of ghost cells for each array + int Nghost = 1; + + // Ncomp = number of components for each array + int Ncomp = 1; + + MultiFab Gamma(ba, dm, Ncomp, Nghost); + + Array P_old; + for (int dir = 0; dir < AMREX_SPACEDIM; dir++) + { + P_old[dir].define(ba, dm, Ncomp, Nghost); + } + + Array P_new; + for (int dir = 0; dir < AMREX_SPACEDIM; dir++) + { + P_new[dir].define(ba, dm, Ncomp, Nghost); + } + + Array P_new_pre; + for (int dir = 0; dir < AMREX_SPACEDIM; dir++) + { + P_new_pre[dir].define(ba, dm, Ncomp, Nghost); + } + + Array GL_rhs; + for (int dir = 0; dir < AMREX_SPACEDIM; dir++) + { + GL_rhs[dir].define(ba, dm, Ncomp, Nghost); + } + + Array GL_rhs_pre; + for (int dir = 0; dir < AMREX_SPACEDIM; dir++) + { + GL_rhs_pre[dir].define(ba, dm, Ncomp, Nghost); + } + + Array GL_rhs_avg; + for (int dir = 0; dir < AMREX_SPACEDIM; dir++) + { + GL_rhs_avg[dir].define(ba, dm, Ncomp, Nghost); + } + + MultiFab PoissonRHS(ba, dm, 1, 0); + MultiFab PoissonPhi(ba, dm, 1, 1); + MultiFab PoissonPhi_Old(ba, dm, 1, 1); + MultiFab PoissonPhi_Prev(ba, dm, 1, 1); + MultiFab PhiErr(ba, dm, 1, 1); + MultiFab Phidiff(ba, dm, 1, 1); + MultiFab Ex(ba, dm, 1, 0); + MultiFab Ey(ba, dm, 1, 0); + MultiFab Ez(ba, dm, 1, 0); + + MultiFab hole_den(ba, dm, 1, 0); + MultiFab e_den(ba, dm, 1, 0); + MultiFab charge_den(ba, dm, 1, 0); + MultiFab MaterialMask(ba, dm, 1, 1); + + //Initialize material mask + InitializeMaterialMask(MaterialMask, geom, prob_lo, prob_hi); + //InitializeMaterialMask(rFerroX, geom, MaterialMask); + + //Solver for Poisson equation + LPInfo info; + std::unique_ptr pMLMG; + // order of stencil + int linop_maxorder = 2; + std::array,2> LinOpBCType_2d; + bool all_homogeneous_boundaries = true; + bool some_functionbased_inhomogeneous_boundaries = false; + bool some_constant_inhomogeneous_boundaries = false; + + bool contains_SC = false; + + FerroX_Util::Contains_sc(MaterialMask, contains_SC); + amrex::Print() << "contains_SC = " << contains_SC << "\n"; + +#ifdef AMREX_USE_EB + MultiFab Plt(ba, dm, 14, 0, MFInfo(), *rGprop.pEB->p_factory_union); +#else + MultiFab Plt(ba, dm, 14, 0); +#endif + + SetPoissonBC(rFerroX, LinOpBCType_2d, all_homogeneous_boundaries, some_functionbased_inhomogeneous_boundaries, some_constant_inhomogeneous_boundaries); + + // coefficients for solver + MultiFab alpha_cc(ba, dm, 1, 0); + MultiFab beta_cc(ba, dm, 1, 1); + std::array< MultiFab, AMREX_SPACEDIM > beta_face; + AMREX_D_TERM(beta_face[0].define(convert(ba,IntVect(AMREX_D_DECL(1,0,0))), dm, 1, 0);, + beta_face[1].define(convert(ba,IntVect(AMREX_D_DECL(0,1,0))), dm, 1, 0);, + beta_face[2].define(convert(ba,IntVect(AMREX_D_DECL(0,0,1))), dm, 1, 0);); + + // set cell-centered beta coefficient to permittivity based on mask + InitializePermittivity(LinOpBCType_2d, beta_cc, MaterialMask, n_cell, geom); + eXstatic_MFab_Util::AverageCellCenteredMultiFabToCellFaces(beta_cc, beta_face); + + int amrlev = 0; //refers to the setcoarsest level of the solve + // time = starting time in the simulation + Real time = 0.0; + +#ifdef AMREX_USE_EB + + std::unique_ptr p_mlebabec; + p_mlebabec = std::make_unique(); + p_mlebabec->define({geom}, {ba}, {dm}, info,{& *rGprop.pEB->p_factory_union}); + + // Force singular system to be solvable + p_mlebabec->setEnforceSingularSolvable(false); + + // set order of stencil + p_mlebabec->setMaxOrder(linop_maxorder); + + // assign domain boundary conditions to the solver + p_mlebabec->setDomainBC(LinOpBCType_2d[0], LinOpBCType_2d[1]); + + if(some_constant_inhomogeneous_boundaries) + { + Fill_Constant_Inhomogeneous_Boundaries(rFerroX, PoissonPhi); + } + if(some_functionbased_inhomogeneous_boundaries) + { + Fill_FunctionBased_Inhomogeneous_Boundaries(rFerroX, PoissonPhi, time); + } + PoissonPhi.FillBoundary(geom.periodicity()); + + // Set Dirichlet BC for Phi in z + //SetPhiBC_z(PoissonPhi); + p_mlebabec->setLevelBC(amrlev, &PoissonPhi); + + // (A*alpha_cc - B * div beta grad) phi = rhs + p_mlebabec->setScalars(-1.0, 1.0); // A = -1.0, B = 1.0; solving (-alpha - div beta grad) phi = RHS + p_mlebabec->setBCoeffs(amrlev, amrex::GetArrOfConstPtrs(beta_face)); + + if(rGprop.pEB->specify_inhomogeneous_dirichlet == 0) + { + p_mlebabec->setEBHomogDirichlet(amrlev, beta_cc); + } + else + { + p_mlebabec->setEBDirichlet(amrlev, *rGprop.pEB->p_surf_soln_union, beta_cc); + } + + pMLMG = std::make_unique(*p_mlebabec); + + pMLMG->setVerbose(mlmg_verbosity); +#else + std::unique_ptr p_mlabec; + p_mlabec = std::make_unique(); + p_mlabec->define({geom}, {ba}, {dm}, info); + + //Force singular system to be solvable + p_mlabec->setEnforceSingularSolvable(false); + + p_mlabec->setMaxOrder(linop_maxorder); + + p_mlabec->setDomainBC(LinOpBCType_2d[0], LinOpBCType_2d[1]); + + if(some_constant_inhomogeneous_boundaries) + { + Fill_Constant_Inhomogeneous_Boundaries(rFerroX, PoissonPhi); + } + if(some_functionbased_inhomogeneous_boundaries) + { + Fill_FunctionBased_Inhomogeneous_Boundaries(rFerroX, PoissonPhi, time); + } + PoissonPhi.FillBoundary(geom.periodicity()); + + // set Dirichlet BC by reading in the ghost cell values + p_mlabec->setLevelBC(amrlev, &PoissonPhi); + + // (A*alpha_cc - B * div beta grad) phi = rhs + p_mlabec->setScalars(-1.0, 1.0); // A = -1.0, B = 1.0; solving (-alpha - div beta grad) phi = RHS + p_mlabec->setBCoeffs(amrlev, amrex::GetArrOfConstPtrs(beta_face)); + + //Declare MLMG object + pMLMG = std::make_unique(*p_mlabec); + pMLMG->setVerbose(mlmg_verbosity); +#endif + + + // INITIALIZE P in FE and rho in SC regions + + //InitializePandRho(P_old, Gamma, charge_den, e_den, hole_den, geom, prob_lo, prob_hi);//old + InitializePandRho(P_old, Gamma, charge_den, e_den, hole_den, MaterialMask, geom, prob_lo, prob_hi);//mask based + + + //Obtain self consisten Phi and rho + Real tol = 1.e-5; + Real err = 1.0; + int iter = 0; + + while(err > tol){ + + //Compute RHS of Poisson equation + ComputePoissonRHS(PoissonRHS, P_old, charge_den, MaterialMask, geom); + + dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, geom, prob_lo, prob_hi); + + ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); + +#ifdef AMREX_USE_EB + p_mlebabec->setACoeffs(0, alpha_cc); +#else + p_mlabec->setACoeffs(0, alpha_cc); +#endif + //Initial guess for phi + PoissonPhi.setVal(0.); + + //Poisson Solve + pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); + PoissonPhi.FillBoundary(geom.periodicity()); + + // Calculate rho from Phi in SC region + ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); + + if (contains_SC == 0) { + // no semiconductor region; set error to zero so the while loop terminates + err = 0.; + } else { + + // Calculate Error + if (iter > 0){ + MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); + MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); + err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); + } + + //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration + MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); + + iter = iter + 1; + amrex::Print() << iter << " iterations :: err = " << err << std::endl; + } + } + + amrex::Print() << "\n ========= Self-Consistent Initialization of P and Rho Done! ========== \n"<< iter << " iterations to obtain self consistent Phi with err = " << err << std::endl; + + // Calculate E from Phi + ComputeEfromPhi(PoissonPhi, Ex, Ey, Ez, geom, prob_lo, prob_hi); + + // Write a plotfile of the initial data if plot_int > 0 + if (plot_int > 0) + { + int step = 0; + const std::string& pltfile = amrex::Concatenate("plt",step,8); + MultiFab::Copy(Plt, P_old[0], 0, 0, 1, 0); + MultiFab::Copy(Plt, P_old[1], 0, 1, 1, 0); + MultiFab::Copy(Plt, P_old[2], 0, 2, 1, 0); + MultiFab::Copy(Plt, PoissonPhi, 0, 3, 1, 0); + MultiFab::Copy(Plt, PoissonRHS, 0, 4, 1, 0); + MultiFab::Copy(Plt, Ex, 0, 5, 1, 0); + MultiFab::Copy(Plt, Ey, 0, 6, 1, 0); + MultiFab::Copy(Plt, Ez, 0, 7, 1, 0); + MultiFab::Copy(Plt, hole_den, 0, 8, 1, 0); + MultiFab::Copy(Plt, e_den, 0, 9, 1, 0); + MultiFab::Copy(Plt, charge_den, 0, 10, 1, 0); + MultiFab::Copy(Plt, beta_cc, 0, 11, 1, 0); + MultiFab::Copy(Plt, MaterialMask, 0, 12, 1, 0); + MultiFab::Copy(Plt, Phidiff, 0, 13, 1, 0); +#ifdef AMREX_USE_EB + amrex::EB_WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "PhiDiff"}, geom, time, step); +#else + amrex::WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "PhiDiff"}, geom, time, step); +#endif + } + + amrex::Print() << "\n ========= Advance Steps ========== \n"<< std::endl; + + int steady_state_step = 1000000; //Initialize to a large number. It will be overwritten by the time step at which steady state condition is satidfied + int sign = 1; //change sign to -1*sign whenever abs(Phi_Bc_hi) == Phi_Bc_hi_max to do triangular wave sweep + int num_Vapp = 0; + + + for (int step = 1; step <= nsteps; ++step) + { + Real step_strt_time = ParallelDescriptor::second(); + + // compute f^n = f(P^n,Phi^n) + CalculateTDGL_RHS(GL_rhs, P_old, PoissonPhi, Gamma, MaterialMask, geom, prob_lo, prob_hi); + + // P^{n+1,*} = P^n + dt * f^n + for (int i = 0; i < 3; i++){ + MultiFab::LinComb(P_new_pre[i], 1.0, P_old[i], 0, dt, GL_rhs[i], 0, 0, 1, Nghost); + P_new_pre[i].FillBoundary(geom.periodicity()); + } + + /** + * \brief dst = a*x + b*y + */ +// static void LinComb (MultiFab& dst, +// Real a, +// const MultiFab& x, +// int xcomp, +// Real b, +// const MultiFab& y, +// int ycomp, +// int dstcomp, +// int numcomp, +// int nghost); + + err = 1.0; + iter = 0; + + // iterate to compute Phi^{n+1,*} + while(err > tol){ + + // Compute RHS of Poisson equation + ComputePoissonRHS(PoissonRHS, P_new_pre, charge_den, MaterialMask, geom); + + dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_new_pre, charge_den, e_den, hole_den, MaterialMask, geom, prob_lo, prob_hi); + + ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); + +#ifdef AMREX_USE_EB + p_mlebabec->setACoeffs(0, alpha_cc); +#else + p_mlabec->setACoeffs(0, alpha_cc); +#endif + //Initial guess for phi + PoissonPhi.setVal(0.); + + //Poisson Solve + pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); + + PoissonPhi.FillBoundary(geom.periodicity()); + + // Calculate rho from Phi in SC region + ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); + + if (contains_SC == 0) { + // no semiconductor region; set error to zero so the while loop terminates + err = 0.; + } else { + + // Calculate Error + if (iter > 0){ + MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); + MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); + err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); + } + + //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration + MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); + + iter = iter + 1; + amrex::Print() << iter << " iterations :: err = " << err << std::endl; + } + } + + if (TimeIntegratorOrder == 1) { + + // copy new solution into old solution + for (int i = 0; i < 3; i++){ + MultiFab::Copy(P_old[i], P_new_pre[i], 0, 0, 1, 0); + // fill periodic ghost cells + P_old[i].FillBoundary(geom.periodicity()); + } + + } else { + + // compute f^{n+1,*} = f(P^{n+1,*},Phi^{n+1,*}) + CalculateTDGL_RHS(GL_rhs_pre, P_new_pre, PoissonPhi, Gamma, MaterialMask, geom, prob_lo, prob_hi); + + // P^{n+1} = P^n + dt/2 * f^n + dt/2 * f^{n+1,*} + for (int i = 0; i < 3; i++){ + MultiFab::LinComb(GL_rhs_avg[i], 0.5, GL_rhs[i], 0, 0.5, GL_rhs_pre[i], 0, 0, 1, Nghost); + MultiFab::LinComb(P_new[i], 1.0, P_old[i], 0, dt, GL_rhs_avg[i], 0, 0, 1, Nghost); + } + + err = 1.0; + iter = 0; + + // iterate to compute Phi^{n+1} + while(err > tol){ + + // Compute RHS of Poisson equation + ComputePoissonRHS(PoissonRHS, P_new, charge_den, MaterialMask, geom); + + dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_new, charge_den, e_den, hole_den, MaterialMask, geom, prob_lo, prob_hi); + + ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); + +#ifdef AMREX_USE_EB + p_mlebabec->setACoeffs(0, alpha_cc); +#else + p_mlabec->setACoeffs(0, alpha_cc); +#endif + + //Initial guess for phi + PoissonPhi.setVal(0.); + + //Poisson Solve + pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); + PoissonPhi.FillBoundary(geom.periodicity()); + + // Calculate rho from Phi in SC region + ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); + + if (contains_SC == 0) { + // no semiconductor region; set error to zero so the while loop terminates + err = 0.; + } else { + + // Calculate Error + if (iter > 0){ + MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); + MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); + err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); + } + + //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration + MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); + + iter = iter + 1; + amrex::Print() << iter << " iterations :: err = " << err << std::endl; + } + } + + // copy new solution into old solution + for (int i = 0; i < 3; i++){ + MultiFab::Copy(P_old[i], P_new[i], 0, 0, 1, 0); + // fill periodic ghost cells + P_old[i].FillBoundary(geom.periodicity()); + } + } + + // Check if steady state has reached + MultiFab::Copy(Phidiff, PoissonPhi, 0, 0, 1, 0); + MultiFab::Subtract(Phidiff, PoissonPhi_Old, 0, 0, 1, 0); + Real phi_err = Phidiff.norm0(); + + if (phi_err < phi_tolerance) { + steady_state_step = step; + inc_step = step; + } + + //Copy PoissonPhi to PoissonPhi_Old to calculate difference at the next iteration + MultiFab::Copy(PoissonPhi_Old, PoissonPhi, 0, 0, 1, 0); + + amrex::Print() << "Steady state check : (phi(t) - phi(t-1)).norm0() = " << phi_err << std::endl; + + + + // Calculate E from Phi + ComputeEfromPhi(PoissonPhi, Ex, Ey, Ez, geom, prob_lo, prob_hi); + + + Real step_stop_time = ParallelDescriptor::second() - step_strt_time; + ParallelDescriptor::ReduceRealMax(step_stop_time); + + amrex::Print() << "Advanced step " << step << " in " << step_stop_time << " seconds\n"; + amrex::Print() << " \n"; + + // update time + time = time + dt; + + + // Write a plotfile of the current data (plot_int was defined in the inputs file) + if (plot_int > 0 && (step%plot_int == 0 || step == steady_state_step)) + { + const std::string& pltfile = amrex::Concatenate("plt",step,8); + MultiFab::Copy(Plt, P_old[0], 0, 0, 1, 0); + MultiFab::Copy(Plt, P_old[1], 0, 1, 1, 0); + MultiFab::Copy(Plt, P_old[2], 0, 2, 1, 0); + MultiFab::Copy(Plt, PoissonPhi, 0, 3, 1, 0); + MultiFab::Copy(Plt, PoissonRHS, 0, 4, 1, 0); + MultiFab::Copy(Plt, Ex, 0, 5, 1, 0); + MultiFab::Copy(Plt, Ey, 0, 6, 1, 0); + MultiFab::Copy(Plt, Ez, 0, 7, 1, 0); + MultiFab::Copy(Plt, hole_den, 0, 8, 1, 0); + MultiFab::Copy(Plt, e_den, 0, 9, 1, 0); + MultiFab::Copy(Plt, charge_den, 0, 10, 1, 0); + + MultiFab::Copy(Plt, beta_cc, 0, 11, 1, 0); + MultiFab::Copy(Plt, MaterialMask, 0, 12, 1, 0); + MultiFab::Copy(Plt, Phidiff, 0, 12, 1, 0); +#ifdef AMREX_USE_EB + amrex::EB_WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "PhiDiff"}, geom, time, step); +#else + amrex::WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "PhiDiff"}, geom, time, step); +#endif + } + + if(voltage_sweep == 1 && inc_step > 0 && step == inc_step) + { + //Update time-dependen Boundary Condition of Poisson's equation + + Phi_Bc_hi += sign*Phi_Bc_inc; + num_Vapp += 1; + + amrex::Print() << "step = " << step << ", Phi_Bc_hi = " << Phi_Bc_hi << ", num_Vapp = " << num_Vapp << std::endl; + + amrex::Print() << "Applied voltage updated at time " << time << ", step = " << step << "\n"; + + // Set Dirichlet BC for Phi in z + SetPhiBC_z(PoissonPhi, n_cell); + + // set Dirichlet BC by reading in the ghost cell values +#ifdef AMREX_USE_EB + p_mlebabec->setLevelBC(amrlev, &PoissonPhi); +#else + p_mlabec->setLevelBC(amrlev, &PoissonPhi); +#endif + + err = 1.0; + iter = 0; + + // iterate to compute Phi^{n+1} with new Dirichlet value + while(err > tol){ + + // Compute RHS of Poisson equation + ComputePoissonRHS(PoissonRHS, P_old, charge_den, MaterialMask, geom); + + dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, geom, prob_lo, prob_hi); + + ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); + +#ifdef AMREX_USE_EB + p_mlebabec->setACoeffs(0, alpha_cc); +#else + p_mlabec->setACoeffs(0, alpha_cc); +#endif + + //Initial guess for phi + PoissonPhi.setVal(0.); + + //Poisson Solve + pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); + PoissonPhi.FillBoundary(geom.periodicity()); + + // Calculate rho from Phi in SC region + ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); + + if (contains_SC == 0) { + // no semiconductor region; set error to zero so the while loop terminates + err = 0.; + } else { + + // Calculate Error + if (iter > 0){ + MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); + MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); + err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); + } + + //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration + MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); + + iter = iter + 1; + amrex::Print() << iter << " iterations :: err = " << err << std::endl; + } + } + + }//end inc_step + + if (voltage_sweep == 1 && step == steady_state_step && std::abs(Phi_Bc_hi) == Phi_Bc_hi_max) sign *= -1; + if (voltage_sweep == 0 && step == steady_state_step) break; + if (voltage_sweep == 1 && Phi_Bc_hi > Phi_Bc_hi_max) break; + if (voltage_sweep == 1 && num_Vapp == 21) break; + + } // end step + + // MultiFab memory usage + const int IOProc = ParallelDescriptor::IOProcessorNumber(); + + amrex::Long min_fab_megabytes = amrex::TotalBytesAllocatedInFabsHWM()/1048576; + amrex::Long max_fab_megabytes = min_fab_megabytes; + + ParallelDescriptor::ReduceLongMin(min_fab_megabytes, IOProc); + ParallelDescriptor::ReduceLongMax(max_fab_megabytes, IOProc); + + amrex::Print() << "High-water FAB megabyte spread across MPI nodes: [" + << min_fab_megabytes << " ... " << max_fab_megabytes << "]\n"; + + min_fab_megabytes = amrex::TotalBytesAllocatedInFabs()/1048576; + max_fab_megabytes = min_fab_megabytes; + + ParallelDescriptor::ReduceLongMin(min_fab_megabytes, IOProc); + ParallelDescriptor::ReduceLongMax(max_fab_megabytes, IOProc); + + amrex::Print() << "Curent FAB megabyte spread across MPI nodes: [" + << min_fab_megabytes << " ... " << max_fab_megabytes << "]\n"; + + Real total_step_stop_time = ParallelDescriptor::second() - total_step_strt_time; + ParallelDescriptor::ReduceRealMax(total_step_stop_time); + + amrex::Print() << "Total run time " << total_step_stop_time << " seconds\n"; + +} diff --git a/Source/main_vsweep.cpp b/Source/main_vsweep.cpp new file mode 100644 index 0000000..65d5699 --- /dev/null +++ b/Source/main_vsweep.cpp @@ -0,0 +1,696 @@ +#include +#include +#include +#ifdef AMREX_USE_EB +#include +#endif +#include +#include +#include +#include "FerroX.H" +#include "Solver/ElectrostaticSolver.H" +#include "Solver/Initialization.H" +#include "Solver/ChargeDensity.H" +#include "Solver/TotalEnergyDensity.H" +#include "Input/BoundaryConditions/BoundaryConditions.H" +#include "Input/GeometryProperties/GeometryProperties.H" +#include "Utils/SelectWarpXUtils/WarpXUtil.H" +#include "Utils/SelectWarpXUtils/WarpXProfilerWrapper.H" +#include "Utils/eXstaticUtils/eXstaticUtil.H" +#include "Utils/FerroXUtils/FerroXUtil.H" + + + + +using namespace amrex; + +using namespace FerroX; + +int main (int argc, char* argv[]) +{ + amrex::Initialize(argc,argv); + + { + c_FerroX pFerroX; + pFerroX.InitData(); + main_main(pFerroX); + } + amrex::Finalize(); + return 0; +} + +void main_main (c_FerroX& rFerroX) +{ + + Real total_step_strt_time = ParallelDescriptor::second(); + + auto& rGprop = rFerroX.get_GeometryProperties(); + auto& geom = rGprop.geom; + auto& ba = rGprop.ba; + auto& dm = rGprop.dm; + auto& is_periodic = rGprop.is_periodic; + auto& prob_lo = rGprop.prob_lo; + auto& prob_hi = rGprop.prob_hi; + auto& n_cell = rGprop.n_cell; + + // read in inputs file + InitializeFerroXNamespace(prob_lo, prob_hi); + + // Nghost = number of ghost cells for each array + int Nghost = 1; + + // Ncomp = number of components for each array + int Ncomp = 1; + + MultiFab Gamma(ba, dm, Ncomp, Nghost); + + Array P_old; + for (int dir = 0; dir < AMREX_SPACEDIM; dir++) + { + P_old[dir].define(ba, dm, Ncomp, Nghost); + } + + Array P_new; + for (int dir = 0; dir < AMREX_SPACEDIM; dir++) + { + P_new[dir].define(ba, dm, Ncomp, Nghost); + } + + Array P_new_pre; + for (int dir = 0; dir < AMREX_SPACEDIM; dir++) + { + P_new_pre[dir].define(ba, dm, Ncomp, Nghost); + } + + Array GL_rhs; + for (int dir = 0; dir < AMREX_SPACEDIM; dir++) + { + GL_rhs[dir].define(ba, dm, Ncomp, Nghost); + } + + Array GL_rhs_pre; + for (int dir = 0; dir < AMREX_SPACEDIM; dir++) + { + GL_rhs_pre[dir].define(ba, dm, Ncomp, Nghost); + } + + Array GL_rhs_avg; + for (int dir = 0; dir < AMREX_SPACEDIM; dir++) + { + GL_rhs_avg[dir].define(ba, dm, Ncomp, Nghost); + } + + Array E; + for (int dir = 0; dir < AMREX_SPACEDIM; dir++) + { + E[dir].define(ba, dm, Ncomp, 0); + } + + MultiFab PoissonRHS(ba, dm, 1, 0); + MultiFab PoissonPhi(ba, dm, 1, 1); + MultiFab PoissonPhi_Old(ba, dm, 1, 1); + MultiFab PoissonPhi_Prev(ba, dm, 1, 1); + MultiFab PhiErr(ba, dm, 1, 1); + MultiFab Phidiff(ba, dm, 1, 1); + MultiFab Ex(ba, dm, 1, 0); + MultiFab Ey(ba, dm, 1, 0); + MultiFab Ez(ba, dm, 1, 0); + + MultiFab hole_den(ba, dm, 1, 0); + MultiFab e_den(ba, dm, 1, 0); + MultiFab charge_den(ba, dm, 1, 0); + MultiFab MaterialMask(ba, dm, 1, 1); + MultiFab tphaseMask(ba, dm, 1, 1); + MultiFab angle_alpha(ba, dm, 1, 0); + MultiFab angle_beta(ba, dm, 1, 0); + MultiFab angle_theta(ba, dm, 1, 0); + + //Initialize material mask + InitializeMaterialMask(MaterialMask, geom, prob_lo, prob_hi); + //InitializeMaterialMask(rFerroX, geom, MaterialMask); + if(Coordinate_Transformation == 1){ + Initialize_tphase_Mask(rFerroX, geom, tphaseMask); + Initialize_Euler_angles(rFerroX, geom, angle_alpha, angle_beta, angle_theta); + } else { + tphaseMask.setVal(0.); + angle_alpha.setVal(0.); + angle_beta.setVal(0.); + angle_theta.setVal(0.); + } + + //Solver for Poisson equation + LPInfo info; + std::unique_ptr pMLMG; + // order of stencil + int linop_maxorder = 2; + std::array,2> LinOpBCType_2d; + bool all_homogeneous_boundaries = true; + bool some_functionbased_inhomogeneous_boundaries = false; + bool some_constant_inhomogeneous_boundaries = false; + + bool contains_SC = false; + + FerroX_Util::Contains_sc(MaterialMask, contains_SC); + amrex::Print() << "contains_SC = " << contains_SC << "\n"; + +#ifdef AMREX_USE_EB + MultiFab Plt(ba, dm, 18, 0, MFInfo(), *rGprop.pEB->p_factory_union); +#else + MultiFab Plt(ba, dm, 18, 0); +#endif + + SetPoissonBC(rFerroX, LinOpBCType_2d, all_homogeneous_boundaries, some_functionbased_inhomogeneous_boundaries, some_constant_inhomogeneous_boundaries); + + // coefficients for solver + MultiFab alpha_cc(ba, dm, 1, 0); + MultiFab beta_cc(ba, dm, 1, 1); + std::array< MultiFab, AMREX_SPACEDIM > beta_face; + AMREX_D_TERM(beta_face[0].define(convert(ba,IntVect(AMREX_D_DECL(1,0,0))), dm, 1, 0);, + beta_face[1].define(convert(ba,IntVect(AMREX_D_DECL(0,1,0))), dm, 1, 0);, + beta_face[2].define(convert(ba,IntVect(AMREX_D_DECL(0,0,1))), dm, 1, 0);); + + // set cell-centered beta coefficient to permittivity based on mask + InitializePermittivity(LinOpBCType_2d, beta_cc, MaterialMask, tphaseMask, n_cell, geom, prob_lo, prob_hi); + eXstatic_MFab_Util::AverageCellCenteredMultiFabToCellFaces(beta_cc, beta_face); + + int amrlev = 0; //refers to the setcoarsest level of the solve + // time = starting time in the simulation + Real time = 0.0; + +#ifdef AMREX_USE_EB + + std::unique_ptr p_mlebabec; + p_mlebabec = std::make_unique(); + p_mlebabec->define({geom}, {ba}, {dm}, info,{& *rGprop.pEB->p_factory_union}); + + // Force singular system to be solvable + p_mlebabec->setEnforceSingularSolvable(false); + + // set order of stencil + p_mlebabec->setMaxOrder(linop_maxorder); + + // assign domain boundary conditions to the solver + p_mlebabec->setDomainBC(LinOpBCType_2d[0], LinOpBCType_2d[1]); + + if(some_constant_inhomogeneous_boundaries) + { + Fill_Constant_Inhomogeneous_Boundaries(rFerroX, PoissonPhi); + } + if(some_functionbased_inhomogeneous_boundaries) + { + Fill_FunctionBased_Inhomogeneous_Boundaries(rFerroX, PoissonPhi, time); + } + PoissonPhi.FillBoundary(geom.periodicity()); + + // Set Dirichlet BC for Phi in z + //SetPhiBC_z(PoissonPhi); + p_mlebabec->setLevelBC(amrlev, &PoissonPhi); + + // (A*alpha_cc - B * div beta grad) phi = rhs + p_mlebabec->setScalars(-1.0, 1.0); // A = -1.0, B = 1.0; solving (-alpha - div beta grad) phi = RHS + p_mlebabec->setBCoeffs(amrlev, amrex::GetArrOfConstPtrs(beta_face)); + + if(rGprop.pEB->specify_inhomogeneous_dirichlet == 0) + { + p_mlebabec->setEBHomogDirichlet(amrlev, beta_cc); + } + else + { + p_mlebabec->setEBDirichlet(amrlev, *rGprop.pEB->p_surf_soln_union, beta_cc); + } + + pMLMG = std::make_unique(*p_mlebabec); + + pMLMG->setVerbose(mlmg_verbosity); +#else + std::unique_ptr p_mlabec; + p_mlabec = std::make_unique(); + p_mlabec->define({geom}, {ba}, {dm}, info); + + //Force singular system to be solvable + p_mlabec->setEnforceSingularSolvable(false); + + p_mlabec->setMaxOrder(linop_maxorder); + + p_mlabec->setDomainBC(LinOpBCType_2d[0], LinOpBCType_2d[1]); + + if(some_constant_inhomogeneous_boundaries) + { + Fill_Constant_Inhomogeneous_Boundaries(rFerroX, PoissonPhi); + } + if(some_functionbased_inhomogeneous_boundaries) + { + Fill_FunctionBased_Inhomogeneous_Boundaries(rFerroX, PoissonPhi, time); + } + PoissonPhi.FillBoundary(geom.periodicity()); + + // set Dirichlet BC by reading in the ghost cell values + p_mlabec->setLevelBC(amrlev, &PoissonPhi); + + // (A*alpha_cc - B * div beta grad) phi = rhs + p_mlabec->setScalars(-1.0, 1.0); // A = -1.0, B = 1.0; solving (-alpha - div beta grad) phi = RHS + p_mlabec->setBCoeffs(amrlev, amrex::GetArrOfConstPtrs(beta_face)); + + //Declare MLMG object + pMLMG = std::make_unique(*p_mlabec); + pMLMG->setVerbose(mlmg_verbosity); +#endif + + + // INITIALIZE P in FE and rho in SC regions + + //InitializePandRho(P_old, Gamma, charge_den, e_den, hole_den, geom, prob_lo, prob_hi);//old + InitializePandRho(P_old, Gamma, charge_den, e_den, hole_den, MaterialMask, tphaseMask, n_cell, geom, prob_lo, prob_hi);//mask based + + //Obtain self consisten Phi and rho + Real tol = 1.e-5; + Real err = 1.0; + int iter = 0; + + while(err > tol){ + + //Compute RHS of Poisson equation + ComputePoissonRHS(PoissonRHS, P_old, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + + dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); + + ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); + +#ifdef AMREX_USE_EB + p_mlebabec->setACoeffs(0, alpha_cc); +#else + p_mlabec->setACoeffs(0, alpha_cc); +#endif + //Initial guess for phi + PoissonPhi.setVal(0.); + + //Poisson Solve + pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); + PoissonPhi.FillBoundary(geom.periodicity()); + + // Calculate rho from Phi in SC region + ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); + + if (contains_SC == 0) { + // no semiconductor region; set error to zero so the while loop terminates + err = 0.; + } else { + + // Calculate Error + if (iter > 0){ + MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); + MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); + err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); + } + + //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration + MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); + + iter = iter + 1; + amrex::Print() << iter << " iterations :: err = " << err << std::endl; + } + } + + amrex::Print() << "\n ========= Self-Consistent Initialization of P and Rho Done! ========== \n"<< iter << " iterations to obtain self consistent Phi with err = " << err << std::endl; + + // Calculate E from Phi + ComputeEfromPhi(PoissonPhi, E, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); + + // Write a plotfile of the initial data if plot_int > 0 + if (plot_int > 0) + { + int step = 0; + const std::string& pltfile = amrex::Concatenate("plt",step,8); + MultiFab::Copy(Plt, P_old[0], 0, 0, 1, 0); + MultiFab::Copy(Plt, P_old[1], 0, 1, 1, 0); + MultiFab::Copy(Plt, P_old[2], 0, 2, 1, 0); + MultiFab::Copy(Plt, PoissonPhi, 0, 3, 1, 0); + MultiFab::Copy(Plt, PoissonRHS, 0, 4, 1, 0); + MultiFab::Copy(Plt, E[0], 0, 5, 1, 0); + MultiFab::Copy(Plt, E[1], 0, 6, 1, 0); + MultiFab::Copy(Plt, E[2], 0, 7, 1, 0); + MultiFab::Copy(Plt, hole_den, 0, 8, 1, 0); + MultiFab::Copy(Plt, e_den, 0, 9, 1, 0); + MultiFab::Copy(Plt, charge_den, 0, 10, 1, 0); + + MultiFab::Copy(Plt, beta_cc, 0, 11, 1, 0); + MultiFab::Copy(Plt, MaterialMask, 0, 12, 1, 0); + MultiFab::Copy(Plt, tphaseMask, 0, 13, 1, 0); + MultiFab::Copy(Plt, angle_alpha, 0, 14, 1, 0); + MultiFab::Copy(Plt, angle_beta, 0, 15, 1, 0); + MultiFab::Copy(Plt, angle_theta, 0, 16, 1, 0); + MultiFab::Copy(Plt, Phidiff, 0, 17, 1, 0); +#ifdef AMREX_USE_EB + amrex::EB_WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "tphase","alpha", "beta", "theta", "PhiDiff"}, geom, time, step); +#else + amrex::WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "tphase","alpha", "beta", "theta", "PhiDiff"}, geom, time, step); +#endif + } + + amrex::Print() << "\n ========= Advance Steps ========== \n"<< std::endl; + + int steady_state_step = 1000000; //Initialize to a large number. It will be overwritten by the time step at which steady state condition is satidfied + + int sign = 1; //change sign to -1*sign whenever abs(Phi_Bc_hi) == Phi_Bc_hi_max to do triangular wave sweep + int num_Vapp = 0; + Real initial_Vapp = Phi_Bc_hi; + + for (int step = 1; step <= nsteps; ++step) + { + Real step_strt_time = ParallelDescriptor::second(); + + // compute f^n = f(P^n,Phi^n) + CalculateTDGL_RHS(GL_rhs, P_old, E, Gamma, MaterialMask, tphaseMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); + + // P^{n+1,*} = P^n + dt * f^n + for (int i = 0; i < 3; i++){ + MultiFab::LinComb(P_new_pre[i], 1.0, P_old[i], 0, dt, GL_rhs[i], 0, 0, 1, Nghost); + P_new_pre[i].FillBoundary(geom.periodicity()); + } + + /** + * \brief dst = a*x + b*y + */ +// static void LinComb (MultiFab& dst, +// Real a, +// const MultiFab& x, +// int xcomp, +// Real b, +// const MultiFab& y, +// int ycomp, +// int dstcomp, +// int numcomp, +// int nghost); + + + err = 1.0; + iter = 0; + + // iterate to compute Phi^{n+1,*} + while(err > tol){ + + // Compute RHS of Poisson equation + ComputePoissonRHS(PoissonRHS, P_new_pre, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + + dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_new_pre, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); + + ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); + +#ifdef AMREX_USE_EB + p_mlebabec->setACoeffs(0, alpha_cc); +#else + p_mlabec->setACoeffs(0, alpha_cc); +#endif + //Initial guess for phi + PoissonPhi.setVal(0.); + + //Poisson Solve + pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); + + PoissonPhi.FillBoundary(geom.periodicity()); + + // Calculate rho from Phi in SC region + ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); + + if (contains_SC == 0) { + // no semiconductor region; set error to zero so the while loop terminates + err = 0.; + } else { + + // Calculate Error + if (iter > 0){ + MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); + MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); + err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); + } + + //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration + MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); + + iter = iter + 1; + amrex::Print() << iter << " iterations :: err = " << err << std::endl; + } + } + + if (TimeIntegratorOrder == 1) { + + // copy new solution into old solution + for (int i = 0; i < 3; i++){ + MultiFab::Copy(P_old[i], P_new_pre[i], 0, 0, 1, 0); + // fill periodic ghost cells + P_old[i].FillBoundary(geom.periodicity()); + P_new_pre[i].FillBoundary(geom.periodicity()); + } + + } else { + + // compute f^{n+1,*} = f(P^{n+1,*},Phi^{n+1,*}) + CalculateTDGL_RHS(GL_rhs_pre, P_new_pre, E, Gamma, MaterialMask, tphaseMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); + + // P^{n+1} = P^n + dt/2 * f^n + dt/2 * f^{n+1,*} + for (int i = 0; i < 3; i++){ + MultiFab::LinComb(GL_rhs_avg[i], 0.5, GL_rhs[i], 0, 0.5, GL_rhs_pre[i], 0, 0, 1, Nghost); + MultiFab::LinComb(P_new[i], 1.0, P_old[i], 0, dt, GL_rhs_avg[i], 0, 0, 1, Nghost); + } + + err = 1.0; + iter = 0; + + // iterate to compute Phi^{n+1} + while(err > tol){ + + // Compute RHS of Poisson equation + ComputePoissonRHS(PoissonRHS, P_new, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + + dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_new, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); + + ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); + +#ifdef AMREX_USE_EB + p_mlebabec->setACoeffs(0, alpha_cc); +#else + p_mlabec->setACoeffs(0, alpha_cc); +#endif + + //Initial guess for phi + PoissonPhi.setVal(0.); + + //Poisson Solve + pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); + PoissonPhi.FillBoundary(geom.periodicity()); + + // Calculate rho from Phi in SC region + ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); + + if (contains_SC == 0) { + // no semiconductor region; set error to zero so the while loop terminates + err = 0.; + } else { + + // Calculate Error + if (iter > 0){ + MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); + MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); + err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); + } + + //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration + MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); + + iter = iter + 1; + amrex::Print() << iter << " iterations :: err = " << err << std::endl; + } + } + + // copy new solution into old solution + for (int i = 0; i < 3; i++){ + MultiFab::Copy(P_old[i], P_new[i], 0, 0, 1, 0); + // fill periodic ghost cells + P_old[i].FillBoundary(geom.periodicity()); + } + } + + // Check if steady state has reached + //MultiFab::Copy(Phidiff, PoissonPhi, 0, 0, 1, 0); + //MultiFab::Subtract(Phidiff, PoissonPhi_Old, 0, 0, 1, 0); + + Real phi_max = PoissonPhi_Old.norm0(); + + for (MFIter mfi(PoissonPhi); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.growntilebox(1); + + const Array4& Phi = PoissonPhi.array(mfi); + const Array4& PhiOld = PoissonPhi_Old.array(mfi); + const Array4& Phi_err = Phidiff.array(mfi); + + + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) + { + Phi_err(i,j,k) = amrex::Math::abs(Phi(i,j,k) - PhiOld(i,j,k)) / phi_max; + }); + } + + Real max_phi_err = Phidiff.norm0(); + + if (max_phi_err < phi_tolerance) { + steady_state_step = step; + inc_step = step; + } + + //Copy PoissonPhi to PoissonPhi_Old to calculate difference at the next iteration + MultiFab::Copy(PoissonPhi_Old, PoissonPhi, 0, 0, 1, 0); + + amrex::Print() << "Steady state check : (phi(t) - phi(t-1)).norm0() = " << max_phi_err << std::endl; + + + // Calculate E from Phi + ComputeEfromPhi(PoissonPhi, E, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); + + + Real step_stop_time = ParallelDescriptor::second() - step_strt_time; + ParallelDescriptor::ReduceRealMax(step_stop_time); + + amrex::Print() << "Advanced step " << step << " in " << step_stop_time << " seconds\n"; + amrex::Print() << " \n"; + + // update time + time = time + dt; + + + // Write a plotfile of the current data (plot_int was defined in the inputs file) + if (plot_int > 0 && (step%plot_int == 0 || step == steady_state_step)) + { + const std::string& pltfile = amrex::Concatenate("plt",step,8); + MultiFab::Copy(Plt, P_old[0], 0, 0, 1, 0); + MultiFab::Copy(Plt, P_old[1], 0, 1, 1, 0); + MultiFab::Copy(Plt, P_old[2], 0, 2, 1, 0); + MultiFab::Copy(Plt, PoissonPhi, 0, 3, 1, 0); + MultiFab::Copy(Plt, PoissonRHS, 0, 4, 1, 0); + MultiFab::Copy(Plt, E[0], 0, 5, 1, 0); + MultiFab::Copy(Plt, E[1], 0, 6, 1, 0); + MultiFab::Copy(Plt, E[2], 0, 7, 1, 0); + MultiFab::Copy(Plt, hole_den, 0,8, 1, 0); + MultiFab::Copy(Plt, e_den, 0, 9, 1, 0); + MultiFab::Copy(Plt, charge_den, 0, 10, 1, 0); + + MultiFab::Copy(Plt, beta_cc, 0, 11, 1, 0); + MultiFab::Copy(Plt, MaterialMask, 0, 12, 1, 0); + MultiFab::Copy(Plt, tphaseMask, 0, 13, 1, 0); + MultiFab::Copy(Plt, angle_alpha, 0, 14, 1, 0); + MultiFab::Copy(Plt, angle_beta, 0, 15, 1, 0); + MultiFab::Copy(Plt, angle_theta, 0, 16, 1, 0); + MultiFab::Copy(Plt, Phidiff, 0, 17, 1, 0); +#ifdef AMREX_USE_EB + amrex::EB_WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "tphase","alpha", "beta", "theta", "PhiDiff"}, geom, time, step); +#else + amrex::WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "tphase","alpha", "beta", "theta", "PhiDiff"}, geom, time, step); +#endif + } + + if(voltage_sweep == 1 && inc_step > 0 && step == inc_step) + { + //Update time-dependent Boundary Condition of Poisson's equation + + amrex::Print() << "Applied voltage updated at time " << time << ", step " << step << "\n"; + + Phi_Bc_hi += sign*Phi_Bc_inc; + num_Vapp += 1; + + amrex::Print() << "step = " << step << ", Phi_Bc_hi = " << Phi_Bc_hi << ", num_Vapp = " << num_Vapp << std::endl; + + // Set Dirichlet BC for Phi in z + SetPhiBC_z(PoissonPhi, n_cell); + + // set Dirichlet BC by reading in the ghost cell values +#ifdef AMREX_USE_EB + p_mlebabec->setLevelBC(amrlev, &PoissonPhi); +#else + p_mlabec->setLevelBC(amrlev, &PoissonPhi); +#endif + + err = 1.0; + iter = 0; + + // iterate to compute Phi^{n+1} with new Dirichlet value + while(err > tol){ + + // Compute RHS of Poisson equation + ComputePoissonRHS(PoissonRHS, P_old, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + + dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); + + ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); + +#ifdef AMREX_USE_EB + p_mlebabec->setACoeffs(0, alpha_cc); +#else + p_mlabec->setACoeffs(0, alpha_cc); +#endif + + //Initial guess for phi + PoissonPhi.setVal(0.); + + //Poisson Solve + pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); + PoissonPhi.FillBoundary(geom.periodicity()); + + // Calculate rho from Phi in SC region + ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); + + if (contains_SC == 0) { + // no semiconductor region; set error to zero so the while loop terminates + err = 0.; + } else { + + // Calculate Error + if (iter > 0){ + MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); + MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); + err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); + } + + //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration + MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); + + iter = iter + 1; + amrex::Print() << iter << " iterations :: err = " << err << std::endl; + } + } + + }//end inc_step + + if (voltage_sweep == 1 && step == steady_state_step && std::abs(Phi_Bc_hi) == Phi_Bc_hi_max) sign *= -1; + if (voltage_sweep == 0 && step == steady_state_step) break; + if (voltage_sweep == 1 && Phi_Bc_hi > Phi_Bc_hi_max) break; + if (voltage_sweep == 1 && num_Vapp == 21) break; + + } // end step + + // MultiFab memory usage + const int IOProc = ParallelDescriptor::IOProcessorNumber(); + + amrex::Long min_fab_megabytes = amrex::TotalBytesAllocatedInFabsHWM()/1048576; + amrex::Long max_fab_megabytes = min_fab_megabytes; + + ParallelDescriptor::ReduceLongMin(min_fab_megabytes, IOProc); + ParallelDescriptor::ReduceLongMax(max_fab_megabytes, IOProc); + + amrex::Print() << "High-water FAB megabyte spread across MPI nodes: [" + << min_fab_megabytes << " ... " << max_fab_megabytes << "]\n"; + + min_fab_megabytes = amrex::TotalBytesAllocatedInFabs()/1048576; + max_fab_megabytes = min_fab_megabytes; + + ParallelDescriptor::ReduceLongMin(min_fab_megabytes, IOProc); + ParallelDescriptor::ReduceLongMax(max_fab_megabytes, IOProc); + + amrex::Print() << "Curent FAB megabyte spread across MPI nodes: [" + << min_fab_megabytes << " ... " << max_fab_megabytes << "]\n"; + + Real total_step_stop_time = ParallelDescriptor::second() - total_step_strt_time; + ParallelDescriptor::ReduceRealMax(total_step_stop_time); + + amrex::Print() << "Total run time " << total_step_stop_time << " seconds\n"; + +} From d9a6f2ef4aa92a43ffcaa8250235eefbe78d14cd Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Thu, 24 Aug 2023 16:56:50 -0700 Subject: [PATCH 10/22] remove unnecessary files --- Source/main_sweep.cpp | 650 -------------------------------------- Source/main_vsweep.cpp | 696 ----------------------------------------- 2 files changed, 1346 deletions(-) delete mode 100644 Source/main_sweep.cpp delete mode 100644 Source/main_vsweep.cpp diff --git a/Source/main_sweep.cpp b/Source/main_sweep.cpp deleted file mode 100644 index 5f73b1c..0000000 --- a/Source/main_sweep.cpp +++ /dev/null @@ -1,650 +0,0 @@ - -#include -#include -#include -#ifdef AMREX_USE_EB -#include -#endif -#include -#include -#include -#include "FerroX.H" -#include "Solver/ElectrostaticSolver.H" -#include "Solver/Initialization.H" -#include "Solver/ChargeDensity.H" -#include "Solver/TotalEnergyDensity.H" -#include "Input/BoundaryConditions/BoundaryConditions.H" -#include "Input/GeometryProperties/GeometryProperties.H" -#include "Utils/SelectWarpXUtils/WarpXUtil.H" -#include "Utils/SelectWarpXUtils/WarpXProfilerWrapper.H" -#include "Utils/eXstaticUtils/eXstaticUtil.H" -#include "Utils/FerroXUtils/FerroXUtil.H" - - - - -using namespace amrex; - -using namespace FerroX; - -int main (int argc, char* argv[]) -{ - amrex::Initialize(argc,argv); - - { - c_FerroX pFerroX; - pFerroX.InitData(); - main_main(pFerroX); - } - amrex::Finalize(); - return 0; -} - -void main_main (c_FerroX& rFerroX) -{ - - Real total_step_strt_time = ParallelDescriptor::second(); - - auto& rGprop = rFerroX.get_GeometryProperties(); - auto& geom = rGprop.geom; - auto& ba = rGprop.ba; - auto& dm = rGprop.dm; - auto& is_periodic = rGprop.is_periodic; - auto& prob_lo = rGprop.prob_lo; - auto& prob_hi = rGprop.prob_hi; - auto& n_cell = rGprop.n_cell; - - // read in inputs file - InitializeFerroXNamespace(prob_lo, prob_hi); - - // Nghost = number of ghost cells for each array - int Nghost = 1; - - // Ncomp = number of components for each array - int Ncomp = 1; - - MultiFab Gamma(ba, dm, Ncomp, Nghost); - - Array P_old; - for (int dir = 0; dir < AMREX_SPACEDIM; dir++) - { - P_old[dir].define(ba, dm, Ncomp, Nghost); - } - - Array P_new; - for (int dir = 0; dir < AMREX_SPACEDIM; dir++) - { - P_new[dir].define(ba, dm, Ncomp, Nghost); - } - - Array P_new_pre; - for (int dir = 0; dir < AMREX_SPACEDIM; dir++) - { - P_new_pre[dir].define(ba, dm, Ncomp, Nghost); - } - - Array GL_rhs; - for (int dir = 0; dir < AMREX_SPACEDIM; dir++) - { - GL_rhs[dir].define(ba, dm, Ncomp, Nghost); - } - - Array GL_rhs_pre; - for (int dir = 0; dir < AMREX_SPACEDIM; dir++) - { - GL_rhs_pre[dir].define(ba, dm, Ncomp, Nghost); - } - - Array GL_rhs_avg; - for (int dir = 0; dir < AMREX_SPACEDIM; dir++) - { - GL_rhs_avg[dir].define(ba, dm, Ncomp, Nghost); - } - - MultiFab PoissonRHS(ba, dm, 1, 0); - MultiFab PoissonPhi(ba, dm, 1, 1); - MultiFab PoissonPhi_Old(ba, dm, 1, 1); - MultiFab PoissonPhi_Prev(ba, dm, 1, 1); - MultiFab PhiErr(ba, dm, 1, 1); - MultiFab Phidiff(ba, dm, 1, 1); - MultiFab Ex(ba, dm, 1, 0); - MultiFab Ey(ba, dm, 1, 0); - MultiFab Ez(ba, dm, 1, 0); - - MultiFab hole_den(ba, dm, 1, 0); - MultiFab e_den(ba, dm, 1, 0); - MultiFab charge_den(ba, dm, 1, 0); - MultiFab MaterialMask(ba, dm, 1, 1); - - //Initialize material mask - InitializeMaterialMask(MaterialMask, geom, prob_lo, prob_hi); - //InitializeMaterialMask(rFerroX, geom, MaterialMask); - - //Solver for Poisson equation - LPInfo info; - std::unique_ptr pMLMG; - // order of stencil - int linop_maxorder = 2; - std::array,2> LinOpBCType_2d; - bool all_homogeneous_boundaries = true; - bool some_functionbased_inhomogeneous_boundaries = false; - bool some_constant_inhomogeneous_boundaries = false; - - bool contains_SC = false; - - FerroX_Util::Contains_sc(MaterialMask, contains_SC); - amrex::Print() << "contains_SC = " << contains_SC << "\n"; - -#ifdef AMREX_USE_EB - MultiFab Plt(ba, dm, 14, 0, MFInfo(), *rGprop.pEB->p_factory_union); -#else - MultiFab Plt(ba, dm, 14, 0); -#endif - - SetPoissonBC(rFerroX, LinOpBCType_2d, all_homogeneous_boundaries, some_functionbased_inhomogeneous_boundaries, some_constant_inhomogeneous_boundaries); - - // coefficients for solver - MultiFab alpha_cc(ba, dm, 1, 0); - MultiFab beta_cc(ba, dm, 1, 1); - std::array< MultiFab, AMREX_SPACEDIM > beta_face; - AMREX_D_TERM(beta_face[0].define(convert(ba,IntVect(AMREX_D_DECL(1,0,0))), dm, 1, 0);, - beta_face[1].define(convert(ba,IntVect(AMREX_D_DECL(0,1,0))), dm, 1, 0);, - beta_face[2].define(convert(ba,IntVect(AMREX_D_DECL(0,0,1))), dm, 1, 0);); - - // set cell-centered beta coefficient to permittivity based on mask - InitializePermittivity(LinOpBCType_2d, beta_cc, MaterialMask, n_cell, geom); - eXstatic_MFab_Util::AverageCellCenteredMultiFabToCellFaces(beta_cc, beta_face); - - int amrlev = 0; //refers to the setcoarsest level of the solve - // time = starting time in the simulation - Real time = 0.0; - -#ifdef AMREX_USE_EB - - std::unique_ptr p_mlebabec; - p_mlebabec = std::make_unique(); - p_mlebabec->define({geom}, {ba}, {dm}, info,{& *rGprop.pEB->p_factory_union}); - - // Force singular system to be solvable - p_mlebabec->setEnforceSingularSolvable(false); - - // set order of stencil - p_mlebabec->setMaxOrder(linop_maxorder); - - // assign domain boundary conditions to the solver - p_mlebabec->setDomainBC(LinOpBCType_2d[0], LinOpBCType_2d[1]); - - if(some_constant_inhomogeneous_boundaries) - { - Fill_Constant_Inhomogeneous_Boundaries(rFerroX, PoissonPhi); - } - if(some_functionbased_inhomogeneous_boundaries) - { - Fill_FunctionBased_Inhomogeneous_Boundaries(rFerroX, PoissonPhi, time); - } - PoissonPhi.FillBoundary(geom.periodicity()); - - // Set Dirichlet BC for Phi in z - //SetPhiBC_z(PoissonPhi); - p_mlebabec->setLevelBC(amrlev, &PoissonPhi); - - // (A*alpha_cc - B * div beta grad) phi = rhs - p_mlebabec->setScalars(-1.0, 1.0); // A = -1.0, B = 1.0; solving (-alpha - div beta grad) phi = RHS - p_mlebabec->setBCoeffs(amrlev, amrex::GetArrOfConstPtrs(beta_face)); - - if(rGprop.pEB->specify_inhomogeneous_dirichlet == 0) - { - p_mlebabec->setEBHomogDirichlet(amrlev, beta_cc); - } - else - { - p_mlebabec->setEBDirichlet(amrlev, *rGprop.pEB->p_surf_soln_union, beta_cc); - } - - pMLMG = std::make_unique(*p_mlebabec); - - pMLMG->setVerbose(mlmg_verbosity); -#else - std::unique_ptr p_mlabec; - p_mlabec = std::make_unique(); - p_mlabec->define({geom}, {ba}, {dm}, info); - - //Force singular system to be solvable - p_mlabec->setEnforceSingularSolvable(false); - - p_mlabec->setMaxOrder(linop_maxorder); - - p_mlabec->setDomainBC(LinOpBCType_2d[0], LinOpBCType_2d[1]); - - if(some_constant_inhomogeneous_boundaries) - { - Fill_Constant_Inhomogeneous_Boundaries(rFerroX, PoissonPhi); - } - if(some_functionbased_inhomogeneous_boundaries) - { - Fill_FunctionBased_Inhomogeneous_Boundaries(rFerroX, PoissonPhi, time); - } - PoissonPhi.FillBoundary(geom.periodicity()); - - // set Dirichlet BC by reading in the ghost cell values - p_mlabec->setLevelBC(amrlev, &PoissonPhi); - - // (A*alpha_cc - B * div beta grad) phi = rhs - p_mlabec->setScalars(-1.0, 1.0); // A = -1.0, B = 1.0; solving (-alpha - div beta grad) phi = RHS - p_mlabec->setBCoeffs(amrlev, amrex::GetArrOfConstPtrs(beta_face)); - - //Declare MLMG object - pMLMG = std::make_unique(*p_mlabec); - pMLMG->setVerbose(mlmg_verbosity); -#endif - - - // INITIALIZE P in FE and rho in SC regions - - //InitializePandRho(P_old, Gamma, charge_den, e_den, hole_den, geom, prob_lo, prob_hi);//old - InitializePandRho(P_old, Gamma, charge_den, e_den, hole_den, MaterialMask, geom, prob_lo, prob_hi);//mask based - - - //Obtain self consisten Phi and rho - Real tol = 1.e-5; - Real err = 1.0; - int iter = 0; - - while(err > tol){ - - //Compute RHS of Poisson equation - ComputePoissonRHS(PoissonRHS, P_old, charge_den, MaterialMask, geom); - - dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, geom, prob_lo, prob_hi); - - ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); - -#ifdef AMREX_USE_EB - p_mlebabec->setACoeffs(0, alpha_cc); -#else - p_mlabec->setACoeffs(0, alpha_cc); -#endif - //Initial guess for phi - PoissonPhi.setVal(0.); - - //Poisson Solve - pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); - PoissonPhi.FillBoundary(geom.periodicity()); - - // Calculate rho from Phi in SC region - ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); - - if (contains_SC == 0) { - // no semiconductor region; set error to zero so the while loop terminates - err = 0.; - } else { - - // Calculate Error - if (iter > 0){ - MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); - MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); - err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); - } - - //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration - MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); - - iter = iter + 1; - amrex::Print() << iter << " iterations :: err = " << err << std::endl; - } - } - - amrex::Print() << "\n ========= Self-Consistent Initialization of P and Rho Done! ========== \n"<< iter << " iterations to obtain self consistent Phi with err = " << err << std::endl; - - // Calculate E from Phi - ComputeEfromPhi(PoissonPhi, Ex, Ey, Ez, geom, prob_lo, prob_hi); - - // Write a plotfile of the initial data if plot_int > 0 - if (plot_int > 0) - { - int step = 0; - const std::string& pltfile = amrex::Concatenate("plt",step,8); - MultiFab::Copy(Plt, P_old[0], 0, 0, 1, 0); - MultiFab::Copy(Plt, P_old[1], 0, 1, 1, 0); - MultiFab::Copy(Plt, P_old[2], 0, 2, 1, 0); - MultiFab::Copy(Plt, PoissonPhi, 0, 3, 1, 0); - MultiFab::Copy(Plt, PoissonRHS, 0, 4, 1, 0); - MultiFab::Copy(Plt, Ex, 0, 5, 1, 0); - MultiFab::Copy(Plt, Ey, 0, 6, 1, 0); - MultiFab::Copy(Plt, Ez, 0, 7, 1, 0); - MultiFab::Copy(Plt, hole_den, 0, 8, 1, 0); - MultiFab::Copy(Plt, e_den, 0, 9, 1, 0); - MultiFab::Copy(Plt, charge_den, 0, 10, 1, 0); - MultiFab::Copy(Plt, beta_cc, 0, 11, 1, 0); - MultiFab::Copy(Plt, MaterialMask, 0, 12, 1, 0); - MultiFab::Copy(Plt, Phidiff, 0, 13, 1, 0); -#ifdef AMREX_USE_EB - amrex::EB_WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "PhiDiff"}, geom, time, step); -#else - amrex::WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "PhiDiff"}, geom, time, step); -#endif - } - - amrex::Print() << "\n ========= Advance Steps ========== \n"<< std::endl; - - int steady_state_step = 1000000; //Initialize to a large number. It will be overwritten by the time step at which steady state condition is satidfied - int sign = 1; //change sign to -1*sign whenever abs(Phi_Bc_hi) == Phi_Bc_hi_max to do triangular wave sweep - int num_Vapp = 0; - - - for (int step = 1; step <= nsteps; ++step) - { - Real step_strt_time = ParallelDescriptor::second(); - - // compute f^n = f(P^n,Phi^n) - CalculateTDGL_RHS(GL_rhs, P_old, PoissonPhi, Gamma, MaterialMask, geom, prob_lo, prob_hi); - - // P^{n+1,*} = P^n + dt * f^n - for (int i = 0; i < 3; i++){ - MultiFab::LinComb(P_new_pre[i], 1.0, P_old[i], 0, dt, GL_rhs[i], 0, 0, 1, Nghost); - P_new_pre[i].FillBoundary(geom.periodicity()); - } - - /** - * \brief dst = a*x + b*y - */ -// static void LinComb (MultiFab& dst, -// Real a, -// const MultiFab& x, -// int xcomp, -// Real b, -// const MultiFab& y, -// int ycomp, -// int dstcomp, -// int numcomp, -// int nghost); - - err = 1.0; - iter = 0; - - // iterate to compute Phi^{n+1,*} - while(err > tol){ - - // Compute RHS of Poisson equation - ComputePoissonRHS(PoissonRHS, P_new_pre, charge_den, MaterialMask, geom); - - dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_new_pre, charge_den, e_den, hole_den, MaterialMask, geom, prob_lo, prob_hi); - - ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); - -#ifdef AMREX_USE_EB - p_mlebabec->setACoeffs(0, alpha_cc); -#else - p_mlabec->setACoeffs(0, alpha_cc); -#endif - //Initial guess for phi - PoissonPhi.setVal(0.); - - //Poisson Solve - pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); - - PoissonPhi.FillBoundary(geom.periodicity()); - - // Calculate rho from Phi in SC region - ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); - - if (contains_SC == 0) { - // no semiconductor region; set error to zero so the while loop terminates - err = 0.; - } else { - - // Calculate Error - if (iter > 0){ - MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); - MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); - err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); - } - - //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration - MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); - - iter = iter + 1; - amrex::Print() << iter << " iterations :: err = " << err << std::endl; - } - } - - if (TimeIntegratorOrder == 1) { - - // copy new solution into old solution - for (int i = 0; i < 3; i++){ - MultiFab::Copy(P_old[i], P_new_pre[i], 0, 0, 1, 0); - // fill periodic ghost cells - P_old[i].FillBoundary(geom.periodicity()); - } - - } else { - - // compute f^{n+1,*} = f(P^{n+1,*},Phi^{n+1,*}) - CalculateTDGL_RHS(GL_rhs_pre, P_new_pre, PoissonPhi, Gamma, MaterialMask, geom, prob_lo, prob_hi); - - // P^{n+1} = P^n + dt/2 * f^n + dt/2 * f^{n+1,*} - for (int i = 0; i < 3; i++){ - MultiFab::LinComb(GL_rhs_avg[i], 0.5, GL_rhs[i], 0, 0.5, GL_rhs_pre[i], 0, 0, 1, Nghost); - MultiFab::LinComb(P_new[i], 1.0, P_old[i], 0, dt, GL_rhs_avg[i], 0, 0, 1, Nghost); - } - - err = 1.0; - iter = 0; - - // iterate to compute Phi^{n+1} - while(err > tol){ - - // Compute RHS of Poisson equation - ComputePoissonRHS(PoissonRHS, P_new, charge_den, MaterialMask, geom); - - dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_new, charge_den, e_den, hole_den, MaterialMask, geom, prob_lo, prob_hi); - - ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); - -#ifdef AMREX_USE_EB - p_mlebabec->setACoeffs(0, alpha_cc); -#else - p_mlabec->setACoeffs(0, alpha_cc); -#endif - - //Initial guess for phi - PoissonPhi.setVal(0.); - - //Poisson Solve - pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); - PoissonPhi.FillBoundary(geom.periodicity()); - - // Calculate rho from Phi in SC region - ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); - - if (contains_SC == 0) { - // no semiconductor region; set error to zero so the while loop terminates - err = 0.; - } else { - - // Calculate Error - if (iter > 0){ - MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); - MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); - err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); - } - - //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration - MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); - - iter = iter + 1; - amrex::Print() << iter << " iterations :: err = " << err << std::endl; - } - } - - // copy new solution into old solution - for (int i = 0; i < 3; i++){ - MultiFab::Copy(P_old[i], P_new[i], 0, 0, 1, 0); - // fill periodic ghost cells - P_old[i].FillBoundary(geom.periodicity()); - } - } - - // Check if steady state has reached - MultiFab::Copy(Phidiff, PoissonPhi, 0, 0, 1, 0); - MultiFab::Subtract(Phidiff, PoissonPhi_Old, 0, 0, 1, 0); - Real phi_err = Phidiff.norm0(); - - if (phi_err < phi_tolerance) { - steady_state_step = step; - inc_step = step; - } - - //Copy PoissonPhi to PoissonPhi_Old to calculate difference at the next iteration - MultiFab::Copy(PoissonPhi_Old, PoissonPhi, 0, 0, 1, 0); - - amrex::Print() << "Steady state check : (phi(t) - phi(t-1)).norm0() = " << phi_err << std::endl; - - - - // Calculate E from Phi - ComputeEfromPhi(PoissonPhi, Ex, Ey, Ez, geom, prob_lo, prob_hi); - - - Real step_stop_time = ParallelDescriptor::second() - step_strt_time; - ParallelDescriptor::ReduceRealMax(step_stop_time); - - amrex::Print() << "Advanced step " << step << " in " << step_stop_time << " seconds\n"; - amrex::Print() << " \n"; - - // update time - time = time + dt; - - - // Write a plotfile of the current data (plot_int was defined in the inputs file) - if (plot_int > 0 && (step%plot_int == 0 || step == steady_state_step)) - { - const std::string& pltfile = amrex::Concatenate("plt",step,8); - MultiFab::Copy(Plt, P_old[0], 0, 0, 1, 0); - MultiFab::Copy(Plt, P_old[1], 0, 1, 1, 0); - MultiFab::Copy(Plt, P_old[2], 0, 2, 1, 0); - MultiFab::Copy(Plt, PoissonPhi, 0, 3, 1, 0); - MultiFab::Copy(Plt, PoissonRHS, 0, 4, 1, 0); - MultiFab::Copy(Plt, Ex, 0, 5, 1, 0); - MultiFab::Copy(Plt, Ey, 0, 6, 1, 0); - MultiFab::Copy(Plt, Ez, 0, 7, 1, 0); - MultiFab::Copy(Plt, hole_den, 0, 8, 1, 0); - MultiFab::Copy(Plt, e_den, 0, 9, 1, 0); - MultiFab::Copy(Plt, charge_den, 0, 10, 1, 0); - - MultiFab::Copy(Plt, beta_cc, 0, 11, 1, 0); - MultiFab::Copy(Plt, MaterialMask, 0, 12, 1, 0); - MultiFab::Copy(Plt, Phidiff, 0, 12, 1, 0); -#ifdef AMREX_USE_EB - amrex::EB_WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "PhiDiff"}, geom, time, step); -#else - amrex::WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "PhiDiff"}, geom, time, step); -#endif - } - - if(voltage_sweep == 1 && inc_step > 0 && step == inc_step) - { - //Update time-dependen Boundary Condition of Poisson's equation - - Phi_Bc_hi += sign*Phi_Bc_inc; - num_Vapp += 1; - - amrex::Print() << "step = " << step << ", Phi_Bc_hi = " << Phi_Bc_hi << ", num_Vapp = " << num_Vapp << std::endl; - - amrex::Print() << "Applied voltage updated at time " << time << ", step = " << step << "\n"; - - // Set Dirichlet BC for Phi in z - SetPhiBC_z(PoissonPhi, n_cell); - - // set Dirichlet BC by reading in the ghost cell values -#ifdef AMREX_USE_EB - p_mlebabec->setLevelBC(amrlev, &PoissonPhi); -#else - p_mlabec->setLevelBC(amrlev, &PoissonPhi); -#endif - - err = 1.0; - iter = 0; - - // iterate to compute Phi^{n+1} with new Dirichlet value - while(err > tol){ - - // Compute RHS of Poisson equation - ComputePoissonRHS(PoissonRHS, P_old, charge_den, MaterialMask, geom); - - dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, geom, prob_lo, prob_hi); - - ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); - -#ifdef AMREX_USE_EB - p_mlebabec->setACoeffs(0, alpha_cc); -#else - p_mlabec->setACoeffs(0, alpha_cc); -#endif - - //Initial guess for phi - PoissonPhi.setVal(0.); - - //Poisson Solve - pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); - PoissonPhi.FillBoundary(geom.periodicity()); - - // Calculate rho from Phi in SC region - ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); - - if (contains_SC == 0) { - // no semiconductor region; set error to zero so the while loop terminates - err = 0.; - } else { - - // Calculate Error - if (iter > 0){ - MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); - MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); - err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); - } - - //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration - MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); - - iter = iter + 1; - amrex::Print() << iter << " iterations :: err = " << err << std::endl; - } - } - - }//end inc_step - - if (voltage_sweep == 1 && step == steady_state_step && std::abs(Phi_Bc_hi) == Phi_Bc_hi_max) sign *= -1; - if (voltage_sweep == 0 && step == steady_state_step) break; - if (voltage_sweep == 1 && Phi_Bc_hi > Phi_Bc_hi_max) break; - if (voltage_sweep == 1 && num_Vapp == 21) break; - - } // end step - - // MultiFab memory usage - const int IOProc = ParallelDescriptor::IOProcessorNumber(); - - amrex::Long min_fab_megabytes = amrex::TotalBytesAllocatedInFabsHWM()/1048576; - amrex::Long max_fab_megabytes = min_fab_megabytes; - - ParallelDescriptor::ReduceLongMin(min_fab_megabytes, IOProc); - ParallelDescriptor::ReduceLongMax(max_fab_megabytes, IOProc); - - amrex::Print() << "High-water FAB megabyte spread across MPI nodes: [" - << min_fab_megabytes << " ... " << max_fab_megabytes << "]\n"; - - min_fab_megabytes = amrex::TotalBytesAllocatedInFabs()/1048576; - max_fab_megabytes = min_fab_megabytes; - - ParallelDescriptor::ReduceLongMin(min_fab_megabytes, IOProc); - ParallelDescriptor::ReduceLongMax(max_fab_megabytes, IOProc); - - amrex::Print() << "Curent FAB megabyte spread across MPI nodes: [" - << min_fab_megabytes << " ... " << max_fab_megabytes << "]\n"; - - Real total_step_stop_time = ParallelDescriptor::second() - total_step_strt_time; - ParallelDescriptor::ReduceRealMax(total_step_stop_time); - - amrex::Print() << "Total run time " << total_step_stop_time << " seconds\n"; - -} diff --git a/Source/main_vsweep.cpp b/Source/main_vsweep.cpp deleted file mode 100644 index 65d5699..0000000 --- a/Source/main_vsweep.cpp +++ /dev/null @@ -1,696 +0,0 @@ -#include -#include -#include -#ifdef AMREX_USE_EB -#include -#endif -#include -#include -#include -#include "FerroX.H" -#include "Solver/ElectrostaticSolver.H" -#include "Solver/Initialization.H" -#include "Solver/ChargeDensity.H" -#include "Solver/TotalEnergyDensity.H" -#include "Input/BoundaryConditions/BoundaryConditions.H" -#include "Input/GeometryProperties/GeometryProperties.H" -#include "Utils/SelectWarpXUtils/WarpXUtil.H" -#include "Utils/SelectWarpXUtils/WarpXProfilerWrapper.H" -#include "Utils/eXstaticUtils/eXstaticUtil.H" -#include "Utils/FerroXUtils/FerroXUtil.H" - - - - -using namespace amrex; - -using namespace FerroX; - -int main (int argc, char* argv[]) -{ - amrex::Initialize(argc,argv); - - { - c_FerroX pFerroX; - pFerroX.InitData(); - main_main(pFerroX); - } - amrex::Finalize(); - return 0; -} - -void main_main (c_FerroX& rFerroX) -{ - - Real total_step_strt_time = ParallelDescriptor::second(); - - auto& rGprop = rFerroX.get_GeometryProperties(); - auto& geom = rGprop.geom; - auto& ba = rGprop.ba; - auto& dm = rGprop.dm; - auto& is_periodic = rGprop.is_periodic; - auto& prob_lo = rGprop.prob_lo; - auto& prob_hi = rGprop.prob_hi; - auto& n_cell = rGprop.n_cell; - - // read in inputs file - InitializeFerroXNamespace(prob_lo, prob_hi); - - // Nghost = number of ghost cells for each array - int Nghost = 1; - - // Ncomp = number of components for each array - int Ncomp = 1; - - MultiFab Gamma(ba, dm, Ncomp, Nghost); - - Array P_old; - for (int dir = 0; dir < AMREX_SPACEDIM; dir++) - { - P_old[dir].define(ba, dm, Ncomp, Nghost); - } - - Array P_new; - for (int dir = 0; dir < AMREX_SPACEDIM; dir++) - { - P_new[dir].define(ba, dm, Ncomp, Nghost); - } - - Array P_new_pre; - for (int dir = 0; dir < AMREX_SPACEDIM; dir++) - { - P_new_pre[dir].define(ba, dm, Ncomp, Nghost); - } - - Array GL_rhs; - for (int dir = 0; dir < AMREX_SPACEDIM; dir++) - { - GL_rhs[dir].define(ba, dm, Ncomp, Nghost); - } - - Array GL_rhs_pre; - for (int dir = 0; dir < AMREX_SPACEDIM; dir++) - { - GL_rhs_pre[dir].define(ba, dm, Ncomp, Nghost); - } - - Array GL_rhs_avg; - for (int dir = 0; dir < AMREX_SPACEDIM; dir++) - { - GL_rhs_avg[dir].define(ba, dm, Ncomp, Nghost); - } - - Array E; - for (int dir = 0; dir < AMREX_SPACEDIM; dir++) - { - E[dir].define(ba, dm, Ncomp, 0); - } - - MultiFab PoissonRHS(ba, dm, 1, 0); - MultiFab PoissonPhi(ba, dm, 1, 1); - MultiFab PoissonPhi_Old(ba, dm, 1, 1); - MultiFab PoissonPhi_Prev(ba, dm, 1, 1); - MultiFab PhiErr(ba, dm, 1, 1); - MultiFab Phidiff(ba, dm, 1, 1); - MultiFab Ex(ba, dm, 1, 0); - MultiFab Ey(ba, dm, 1, 0); - MultiFab Ez(ba, dm, 1, 0); - - MultiFab hole_den(ba, dm, 1, 0); - MultiFab e_den(ba, dm, 1, 0); - MultiFab charge_den(ba, dm, 1, 0); - MultiFab MaterialMask(ba, dm, 1, 1); - MultiFab tphaseMask(ba, dm, 1, 1); - MultiFab angle_alpha(ba, dm, 1, 0); - MultiFab angle_beta(ba, dm, 1, 0); - MultiFab angle_theta(ba, dm, 1, 0); - - //Initialize material mask - InitializeMaterialMask(MaterialMask, geom, prob_lo, prob_hi); - //InitializeMaterialMask(rFerroX, geom, MaterialMask); - if(Coordinate_Transformation == 1){ - Initialize_tphase_Mask(rFerroX, geom, tphaseMask); - Initialize_Euler_angles(rFerroX, geom, angle_alpha, angle_beta, angle_theta); - } else { - tphaseMask.setVal(0.); - angle_alpha.setVal(0.); - angle_beta.setVal(0.); - angle_theta.setVal(0.); - } - - //Solver for Poisson equation - LPInfo info; - std::unique_ptr pMLMG; - // order of stencil - int linop_maxorder = 2; - std::array,2> LinOpBCType_2d; - bool all_homogeneous_boundaries = true; - bool some_functionbased_inhomogeneous_boundaries = false; - bool some_constant_inhomogeneous_boundaries = false; - - bool contains_SC = false; - - FerroX_Util::Contains_sc(MaterialMask, contains_SC); - amrex::Print() << "contains_SC = " << contains_SC << "\n"; - -#ifdef AMREX_USE_EB - MultiFab Plt(ba, dm, 18, 0, MFInfo(), *rGprop.pEB->p_factory_union); -#else - MultiFab Plt(ba, dm, 18, 0); -#endif - - SetPoissonBC(rFerroX, LinOpBCType_2d, all_homogeneous_boundaries, some_functionbased_inhomogeneous_boundaries, some_constant_inhomogeneous_boundaries); - - // coefficients for solver - MultiFab alpha_cc(ba, dm, 1, 0); - MultiFab beta_cc(ba, dm, 1, 1); - std::array< MultiFab, AMREX_SPACEDIM > beta_face; - AMREX_D_TERM(beta_face[0].define(convert(ba,IntVect(AMREX_D_DECL(1,0,0))), dm, 1, 0);, - beta_face[1].define(convert(ba,IntVect(AMREX_D_DECL(0,1,0))), dm, 1, 0);, - beta_face[2].define(convert(ba,IntVect(AMREX_D_DECL(0,0,1))), dm, 1, 0);); - - // set cell-centered beta coefficient to permittivity based on mask - InitializePermittivity(LinOpBCType_2d, beta_cc, MaterialMask, tphaseMask, n_cell, geom, prob_lo, prob_hi); - eXstatic_MFab_Util::AverageCellCenteredMultiFabToCellFaces(beta_cc, beta_face); - - int amrlev = 0; //refers to the setcoarsest level of the solve - // time = starting time in the simulation - Real time = 0.0; - -#ifdef AMREX_USE_EB - - std::unique_ptr p_mlebabec; - p_mlebabec = std::make_unique(); - p_mlebabec->define({geom}, {ba}, {dm}, info,{& *rGprop.pEB->p_factory_union}); - - // Force singular system to be solvable - p_mlebabec->setEnforceSingularSolvable(false); - - // set order of stencil - p_mlebabec->setMaxOrder(linop_maxorder); - - // assign domain boundary conditions to the solver - p_mlebabec->setDomainBC(LinOpBCType_2d[0], LinOpBCType_2d[1]); - - if(some_constant_inhomogeneous_boundaries) - { - Fill_Constant_Inhomogeneous_Boundaries(rFerroX, PoissonPhi); - } - if(some_functionbased_inhomogeneous_boundaries) - { - Fill_FunctionBased_Inhomogeneous_Boundaries(rFerroX, PoissonPhi, time); - } - PoissonPhi.FillBoundary(geom.periodicity()); - - // Set Dirichlet BC for Phi in z - //SetPhiBC_z(PoissonPhi); - p_mlebabec->setLevelBC(amrlev, &PoissonPhi); - - // (A*alpha_cc - B * div beta grad) phi = rhs - p_mlebabec->setScalars(-1.0, 1.0); // A = -1.0, B = 1.0; solving (-alpha - div beta grad) phi = RHS - p_mlebabec->setBCoeffs(amrlev, amrex::GetArrOfConstPtrs(beta_face)); - - if(rGprop.pEB->specify_inhomogeneous_dirichlet == 0) - { - p_mlebabec->setEBHomogDirichlet(amrlev, beta_cc); - } - else - { - p_mlebabec->setEBDirichlet(amrlev, *rGprop.pEB->p_surf_soln_union, beta_cc); - } - - pMLMG = std::make_unique(*p_mlebabec); - - pMLMG->setVerbose(mlmg_verbosity); -#else - std::unique_ptr p_mlabec; - p_mlabec = std::make_unique(); - p_mlabec->define({geom}, {ba}, {dm}, info); - - //Force singular system to be solvable - p_mlabec->setEnforceSingularSolvable(false); - - p_mlabec->setMaxOrder(linop_maxorder); - - p_mlabec->setDomainBC(LinOpBCType_2d[0], LinOpBCType_2d[1]); - - if(some_constant_inhomogeneous_boundaries) - { - Fill_Constant_Inhomogeneous_Boundaries(rFerroX, PoissonPhi); - } - if(some_functionbased_inhomogeneous_boundaries) - { - Fill_FunctionBased_Inhomogeneous_Boundaries(rFerroX, PoissonPhi, time); - } - PoissonPhi.FillBoundary(geom.periodicity()); - - // set Dirichlet BC by reading in the ghost cell values - p_mlabec->setLevelBC(amrlev, &PoissonPhi); - - // (A*alpha_cc - B * div beta grad) phi = rhs - p_mlabec->setScalars(-1.0, 1.0); // A = -1.0, B = 1.0; solving (-alpha - div beta grad) phi = RHS - p_mlabec->setBCoeffs(amrlev, amrex::GetArrOfConstPtrs(beta_face)); - - //Declare MLMG object - pMLMG = std::make_unique(*p_mlabec); - pMLMG->setVerbose(mlmg_verbosity); -#endif - - - // INITIALIZE P in FE and rho in SC regions - - //InitializePandRho(P_old, Gamma, charge_den, e_den, hole_den, geom, prob_lo, prob_hi);//old - InitializePandRho(P_old, Gamma, charge_den, e_den, hole_den, MaterialMask, tphaseMask, n_cell, geom, prob_lo, prob_hi);//mask based - - //Obtain self consisten Phi and rho - Real tol = 1.e-5; - Real err = 1.0; - int iter = 0; - - while(err > tol){ - - //Compute RHS of Poisson equation - ComputePoissonRHS(PoissonRHS, P_old, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); - - dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); - - ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); - -#ifdef AMREX_USE_EB - p_mlebabec->setACoeffs(0, alpha_cc); -#else - p_mlabec->setACoeffs(0, alpha_cc); -#endif - //Initial guess for phi - PoissonPhi.setVal(0.); - - //Poisson Solve - pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); - PoissonPhi.FillBoundary(geom.periodicity()); - - // Calculate rho from Phi in SC region - ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); - - if (contains_SC == 0) { - // no semiconductor region; set error to zero so the while loop terminates - err = 0.; - } else { - - // Calculate Error - if (iter > 0){ - MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); - MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); - err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); - } - - //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration - MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); - - iter = iter + 1; - amrex::Print() << iter << " iterations :: err = " << err << std::endl; - } - } - - amrex::Print() << "\n ========= Self-Consistent Initialization of P and Rho Done! ========== \n"<< iter << " iterations to obtain self consistent Phi with err = " << err << std::endl; - - // Calculate E from Phi - ComputeEfromPhi(PoissonPhi, E, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); - - // Write a plotfile of the initial data if plot_int > 0 - if (plot_int > 0) - { - int step = 0; - const std::string& pltfile = amrex::Concatenate("plt",step,8); - MultiFab::Copy(Plt, P_old[0], 0, 0, 1, 0); - MultiFab::Copy(Plt, P_old[1], 0, 1, 1, 0); - MultiFab::Copy(Plt, P_old[2], 0, 2, 1, 0); - MultiFab::Copy(Plt, PoissonPhi, 0, 3, 1, 0); - MultiFab::Copy(Plt, PoissonRHS, 0, 4, 1, 0); - MultiFab::Copy(Plt, E[0], 0, 5, 1, 0); - MultiFab::Copy(Plt, E[1], 0, 6, 1, 0); - MultiFab::Copy(Plt, E[2], 0, 7, 1, 0); - MultiFab::Copy(Plt, hole_den, 0, 8, 1, 0); - MultiFab::Copy(Plt, e_den, 0, 9, 1, 0); - MultiFab::Copy(Plt, charge_den, 0, 10, 1, 0); - - MultiFab::Copy(Plt, beta_cc, 0, 11, 1, 0); - MultiFab::Copy(Plt, MaterialMask, 0, 12, 1, 0); - MultiFab::Copy(Plt, tphaseMask, 0, 13, 1, 0); - MultiFab::Copy(Plt, angle_alpha, 0, 14, 1, 0); - MultiFab::Copy(Plt, angle_beta, 0, 15, 1, 0); - MultiFab::Copy(Plt, angle_theta, 0, 16, 1, 0); - MultiFab::Copy(Plt, Phidiff, 0, 17, 1, 0); -#ifdef AMREX_USE_EB - amrex::EB_WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "tphase","alpha", "beta", "theta", "PhiDiff"}, geom, time, step); -#else - amrex::WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "tphase","alpha", "beta", "theta", "PhiDiff"}, geom, time, step); -#endif - } - - amrex::Print() << "\n ========= Advance Steps ========== \n"<< std::endl; - - int steady_state_step = 1000000; //Initialize to a large number. It will be overwritten by the time step at which steady state condition is satidfied - - int sign = 1; //change sign to -1*sign whenever abs(Phi_Bc_hi) == Phi_Bc_hi_max to do triangular wave sweep - int num_Vapp = 0; - Real initial_Vapp = Phi_Bc_hi; - - for (int step = 1; step <= nsteps; ++step) - { - Real step_strt_time = ParallelDescriptor::second(); - - // compute f^n = f(P^n,Phi^n) - CalculateTDGL_RHS(GL_rhs, P_old, E, Gamma, MaterialMask, tphaseMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); - - // P^{n+1,*} = P^n + dt * f^n - for (int i = 0; i < 3; i++){ - MultiFab::LinComb(P_new_pre[i], 1.0, P_old[i], 0, dt, GL_rhs[i], 0, 0, 1, Nghost); - P_new_pre[i].FillBoundary(geom.periodicity()); - } - - /** - * \brief dst = a*x + b*y - */ -// static void LinComb (MultiFab& dst, -// Real a, -// const MultiFab& x, -// int xcomp, -// Real b, -// const MultiFab& y, -// int ycomp, -// int dstcomp, -// int numcomp, -// int nghost); - - - err = 1.0; - iter = 0; - - // iterate to compute Phi^{n+1,*} - while(err > tol){ - - // Compute RHS of Poisson equation - ComputePoissonRHS(PoissonRHS, P_new_pre, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); - - dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_new_pre, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); - - ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); - -#ifdef AMREX_USE_EB - p_mlebabec->setACoeffs(0, alpha_cc); -#else - p_mlabec->setACoeffs(0, alpha_cc); -#endif - //Initial guess for phi - PoissonPhi.setVal(0.); - - //Poisson Solve - pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); - - PoissonPhi.FillBoundary(geom.periodicity()); - - // Calculate rho from Phi in SC region - ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); - - if (contains_SC == 0) { - // no semiconductor region; set error to zero so the while loop terminates - err = 0.; - } else { - - // Calculate Error - if (iter > 0){ - MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); - MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); - err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); - } - - //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration - MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); - - iter = iter + 1; - amrex::Print() << iter << " iterations :: err = " << err << std::endl; - } - } - - if (TimeIntegratorOrder == 1) { - - // copy new solution into old solution - for (int i = 0; i < 3; i++){ - MultiFab::Copy(P_old[i], P_new_pre[i], 0, 0, 1, 0); - // fill periodic ghost cells - P_old[i].FillBoundary(geom.periodicity()); - P_new_pre[i].FillBoundary(geom.periodicity()); - } - - } else { - - // compute f^{n+1,*} = f(P^{n+1,*},Phi^{n+1,*}) - CalculateTDGL_RHS(GL_rhs_pre, P_new_pre, E, Gamma, MaterialMask, tphaseMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); - - // P^{n+1} = P^n + dt/2 * f^n + dt/2 * f^{n+1,*} - for (int i = 0; i < 3; i++){ - MultiFab::LinComb(GL_rhs_avg[i], 0.5, GL_rhs[i], 0, 0.5, GL_rhs_pre[i], 0, 0, 1, Nghost); - MultiFab::LinComb(P_new[i], 1.0, P_old[i], 0, dt, GL_rhs_avg[i], 0, 0, 1, Nghost); - } - - err = 1.0; - iter = 0; - - // iterate to compute Phi^{n+1} - while(err > tol){ - - // Compute RHS of Poisson equation - ComputePoissonRHS(PoissonRHS, P_new, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); - - dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_new, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); - - ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); - -#ifdef AMREX_USE_EB - p_mlebabec->setACoeffs(0, alpha_cc); -#else - p_mlabec->setACoeffs(0, alpha_cc); -#endif - - //Initial guess for phi - PoissonPhi.setVal(0.); - - //Poisson Solve - pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); - PoissonPhi.FillBoundary(geom.periodicity()); - - // Calculate rho from Phi in SC region - ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); - - if (contains_SC == 0) { - // no semiconductor region; set error to zero so the while loop terminates - err = 0.; - } else { - - // Calculate Error - if (iter > 0){ - MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); - MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); - err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); - } - - //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration - MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); - - iter = iter + 1; - amrex::Print() << iter << " iterations :: err = " << err << std::endl; - } - } - - // copy new solution into old solution - for (int i = 0; i < 3; i++){ - MultiFab::Copy(P_old[i], P_new[i], 0, 0, 1, 0); - // fill periodic ghost cells - P_old[i].FillBoundary(geom.periodicity()); - } - } - - // Check if steady state has reached - //MultiFab::Copy(Phidiff, PoissonPhi, 0, 0, 1, 0); - //MultiFab::Subtract(Phidiff, PoissonPhi_Old, 0, 0, 1, 0); - - Real phi_max = PoissonPhi_Old.norm0(); - - for (MFIter mfi(PoissonPhi); mfi.isValid(); ++mfi) - { - const Box& bx = mfi.growntilebox(1); - - const Array4& Phi = PoissonPhi.array(mfi); - const Array4& PhiOld = PoissonPhi_Old.array(mfi); - const Array4& Phi_err = Phidiff.array(mfi); - - - amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) - { - Phi_err(i,j,k) = amrex::Math::abs(Phi(i,j,k) - PhiOld(i,j,k)) / phi_max; - }); - } - - Real max_phi_err = Phidiff.norm0(); - - if (max_phi_err < phi_tolerance) { - steady_state_step = step; - inc_step = step; - } - - //Copy PoissonPhi to PoissonPhi_Old to calculate difference at the next iteration - MultiFab::Copy(PoissonPhi_Old, PoissonPhi, 0, 0, 1, 0); - - amrex::Print() << "Steady state check : (phi(t) - phi(t-1)).norm0() = " << max_phi_err << std::endl; - - - // Calculate E from Phi - ComputeEfromPhi(PoissonPhi, E, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); - - - Real step_stop_time = ParallelDescriptor::second() - step_strt_time; - ParallelDescriptor::ReduceRealMax(step_stop_time); - - amrex::Print() << "Advanced step " << step << " in " << step_stop_time << " seconds\n"; - amrex::Print() << " \n"; - - // update time - time = time + dt; - - - // Write a plotfile of the current data (plot_int was defined in the inputs file) - if (plot_int > 0 && (step%plot_int == 0 || step == steady_state_step)) - { - const std::string& pltfile = amrex::Concatenate("plt",step,8); - MultiFab::Copy(Plt, P_old[0], 0, 0, 1, 0); - MultiFab::Copy(Plt, P_old[1], 0, 1, 1, 0); - MultiFab::Copy(Plt, P_old[2], 0, 2, 1, 0); - MultiFab::Copy(Plt, PoissonPhi, 0, 3, 1, 0); - MultiFab::Copy(Plt, PoissonRHS, 0, 4, 1, 0); - MultiFab::Copy(Plt, E[0], 0, 5, 1, 0); - MultiFab::Copy(Plt, E[1], 0, 6, 1, 0); - MultiFab::Copy(Plt, E[2], 0, 7, 1, 0); - MultiFab::Copy(Plt, hole_den, 0,8, 1, 0); - MultiFab::Copy(Plt, e_den, 0, 9, 1, 0); - MultiFab::Copy(Plt, charge_den, 0, 10, 1, 0); - - MultiFab::Copy(Plt, beta_cc, 0, 11, 1, 0); - MultiFab::Copy(Plt, MaterialMask, 0, 12, 1, 0); - MultiFab::Copy(Plt, tphaseMask, 0, 13, 1, 0); - MultiFab::Copy(Plt, angle_alpha, 0, 14, 1, 0); - MultiFab::Copy(Plt, angle_beta, 0, 15, 1, 0); - MultiFab::Copy(Plt, angle_theta, 0, 16, 1, 0); - MultiFab::Copy(Plt, Phidiff, 0, 17, 1, 0); -#ifdef AMREX_USE_EB - amrex::EB_WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "tphase","alpha", "beta", "theta", "PhiDiff"}, geom, time, step); -#else - amrex::WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "tphase","alpha", "beta", "theta", "PhiDiff"}, geom, time, step); -#endif - } - - if(voltage_sweep == 1 && inc_step > 0 && step == inc_step) - { - //Update time-dependent Boundary Condition of Poisson's equation - - amrex::Print() << "Applied voltage updated at time " << time << ", step " << step << "\n"; - - Phi_Bc_hi += sign*Phi_Bc_inc; - num_Vapp += 1; - - amrex::Print() << "step = " << step << ", Phi_Bc_hi = " << Phi_Bc_hi << ", num_Vapp = " << num_Vapp << std::endl; - - // Set Dirichlet BC for Phi in z - SetPhiBC_z(PoissonPhi, n_cell); - - // set Dirichlet BC by reading in the ghost cell values -#ifdef AMREX_USE_EB - p_mlebabec->setLevelBC(amrlev, &PoissonPhi); -#else - p_mlabec->setLevelBC(amrlev, &PoissonPhi); -#endif - - err = 1.0; - iter = 0; - - // iterate to compute Phi^{n+1} with new Dirichlet value - while(err > tol){ - - // Compute RHS of Poisson equation - ComputePoissonRHS(PoissonRHS, P_old, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); - - dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); - - ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); - -#ifdef AMREX_USE_EB - p_mlebabec->setACoeffs(0, alpha_cc); -#else - p_mlabec->setACoeffs(0, alpha_cc); -#endif - - //Initial guess for phi - PoissonPhi.setVal(0.); - - //Poisson Solve - pMLMG->solve({&PoissonPhi}, {&PoissonRHS}, 1.e-10, -1); - PoissonPhi.FillBoundary(geom.periodicity()); - - // Calculate rho from Phi in SC region - ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); - - if (contains_SC == 0) { - // no semiconductor region; set error to zero so the while loop terminates - err = 0.; - } else { - - // Calculate Error - if (iter > 0){ - MultiFab::Copy(PhiErr, PoissonPhi, 0, 0, 1, 0); - MultiFab::Subtract(PhiErr, PoissonPhi_Prev, 0, 0, 1, 0); - err = PhiErr.norm1(0, geom.periodicity())/PoissonPhi.norm1(0, geom.periodicity()); - } - - //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration - MultiFab::Copy(PoissonPhi_Prev, PoissonPhi, 0, 0, 1, 0); - - iter = iter + 1; - amrex::Print() << iter << " iterations :: err = " << err << std::endl; - } - } - - }//end inc_step - - if (voltage_sweep == 1 && step == steady_state_step && std::abs(Phi_Bc_hi) == Phi_Bc_hi_max) sign *= -1; - if (voltage_sweep == 0 && step == steady_state_step) break; - if (voltage_sweep == 1 && Phi_Bc_hi > Phi_Bc_hi_max) break; - if (voltage_sweep == 1 && num_Vapp == 21) break; - - } // end step - - // MultiFab memory usage - const int IOProc = ParallelDescriptor::IOProcessorNumber(); - - amrex::Long min_fab_megabytes = amrex::TotalBytesAllocatedInFabsHWM()/1048576; - amrex::Long max_fab_megabytes = min_fab_megabytes; - - ParallelDescriptor::ReduceLongMin(min_fab_megabytes, IOProc); - ParallelDescriptor::ReduceLongMax(max_fab_megabytes, IOProc); - - amrex::Print() << "High-water FAB megabyte spread across MPI nodes: [" - << min_fab_megabytes << " ... " << max_fab_megabytes << "]\n"; - - min_fab_megabytes = amrex::TotalBytesAllocatedInFabs()/1048576; - max_fab_megabytes = min_fab_megabytes; - - ParallelDescriptor::ReduceLongMin(min_fab_megabytes, IOProc); - ParallelDescriptor::ReduceLongMax(max_fab_megabytes, IOProc); - - amrex::Print() << "Curent FAB megabyte spread across MPI nodes: [" - << min_fab_megabytes << " ... " << max_fab_megabytes << "]\n"; - - Real total_step_stop_time = ParallelDescriptor::second() - total_step_strt_time; - ParallelDescriptor::ReduceRealMax(total_step_stop_time); - - amrex::Print() << "Total run time " << total_step_stop_time << " seconds\n"; - -} From 4d38e7ac2081a9ba193c043b23c783f1e9b96856 Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Mon, 28 Aug 2023 17:57:26 -0700 Subject: [PATCH 11/22] compute rho on cell centers and then average to nodes to get PoissonRHS --- Source/Solver/ChargeDensity.H | 1 + Source/Solver/ChargeDensity.cpp | 39 +++++++++++++--- Source/Solver/ElectrostaticSolver.cpp | 2 +- Source/Solver/Initialization.cpp | 65 +++++++++++++++++++++------ Source/main.cpp | 63 ++++++++++---------------- 5 files changed, 110 insertions(+), 60 deletions(-) diff --git a/Source/Solver/ChargeDensity.H b/Source/Solver/ChargeDensity.H index af92798..308de5e 100644 --- a/Source/Solver/ChargeDensity.H +++ b/Source/Solver/ChargeDensity.H @@ -10,5 +10,6 @@ void ComputeRho(MultiFab& PoissonPhi, MultiFab& rho, MultiFab& e_den, MultiFab& p_den, + const Geometry& geom, const MultiFab& MaterialMask); diff --git a/Source/Solver/ChargeDensity.cpp b/Source/Solver/ChargeDensity.cpp index 101b45f..49869cd 100644 --- a/Source/Solver/ChargeDensity.cpp +++ b/Source/Solver/ChargeDensity.cpp @@ -5,6 +5,7 @@ void ComputeRho(MultiFab& PoissonPhi, MultiFab& rho, MultiFab& e_den, MultiFab& p_den, + const Geometry& geom, const MultiFab& MaterialMask) { // loop over boxes @@ -13,15 +14,10 @@ void ComputeRho(MultiFab& PoissonPhi, const Box& bx = mfi.validbox(); // Calculate charge density from Phi, Nc, Nv, Ec, and Ev - MultiFab acceptor_den(rho.boxArray(), rho.DistributionMap(), 1, 0); - MultiFab donor_den(rho.boxArray(), rho.DistributionMap(), 1, 0); const Array4& hole_den_arr = p_den.array(mfi); const Array4& e_den_arr = e_den.array(mfi); - const Array4& charge_den_arr = rho.array(mfi); const Array4& phi = PoissonPhi.array(mfi); - const Array4& acceptor_den_arr = acceptor_den.array(mfi); - const Array4& donor_den_arr = donor_den.array(mfi); const Array4& mask = MaterialMask.array(mfi); @@ -47,8 +43,6 @@ void ComputeRho(MultiFab& PoissonPhi, hole_den_arr(i,j,k) = 2.0/sqrt(3.14)*Nv*FD_half_p; } else { //Maxwell-Boltzmann - acceptor_den_arr(i,j,k) = acceptor_doping; - donor_den_arr(i,j,k) = 0.0; Real p_0 = acceptor_doping; Real n_0 = intrinsic_carrier_concentration*intrinsic_carrier_concentration/p_0; hole_den_arr(i,j,k) = p_0*exp(-(q*phi(i,j,k))/(kb*T)); @@ -64,6 +58,36 @@ void ComputeRho(MultiFab& PoissonPhi, // donor_den_arr(i,j,k) = donor_doping; //} + + } else { + hole_den_arr(i,j,k) = 0.; + e_den_arr(i,j,k) = 0.; + } + }); + } + + // loop over boxes + for (MFIter mfi(PoissonPhi); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.validbox(); + + // Calculate charge density from Phi, Nc, Nv, Ec, and Ev + MultiFab acceptor_den(rho.boxArray(), rho.DistributionMap(), 1, 0); + MultiFab donor_den(rho.boxArray(), rho.DistributionMap(), 1, 0); + + const Array4& hole_den_arr = p_den.array(mfi); + const Array4& e_den_arr = e_den.array(mfi); + const Array4& charge_den_arr = rho.array(mfi); + const Array4& acceptor_den_arr = acceptor_den.array(mfi); + const Array4& donor_den_arr = donor_den.array(mfi); + const Array4& mask = MaterialMask.array(mfi); + + + amrex::ParallelFor( bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + + if (mask(i,j,k) >= 2.0) { + charge_den_arr(i,j,k) = q*(hole_den_arr(i,j,k) - e_den_arr(i,j,k) - acceptor_den_arr(i,j,k) + donor_den_arr(i,j,k)); } else { @@ -73,4 +97,5 @@ void ComputeRho(MultiFab& PoissonPhi, } }); } + rho.FillBoundary(geom.periodicity()); } diff --git a/Source/Solver/ElectrostaticSolver.cpp b/Source/Solver/ElectrostaticSolver.cpp index 9506b36..29a4f29 100644 --- a/Source/Solver/ElectrostaticSolver.cpp +++ b/Source/Solver/ElectrostaticSolver.cpp @@ -119,7 +119,7 @@ void dF_dPhi(MultiFab& alpha_cc, PoissonPhi_plus_delta.plus(delta, 0, 1, 0); // Calculate rho from Phi in SC region - ComputeRho(PoissonPhi_plus_delta, rho, e_den, p_den, MaterialMask); + ComputeRho(PoissonPhi_plus_delta, rho, e_den, p_den, geom, MaterialMask); //Compute RHS of Poisson equation ComputePoissonRHS(PoissonRHS_phi_plus_delta, P_old, rho, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); diff --git a/Source/Solver/Initialization.cpp b/Source/Solver/Initialization.cpp index a663f3b..9a22d2d 100644 --- a/Source/Solver/Initialization.cpp +++ b/Source/Solver/Initialization.cpp @@ -109,12 +109,15 @@ void InitializePandRho(Array &P_old, pOld_r(i,j,k) = 0.0; } + pOld_p(i,j,k) = 0.0; + pOld_q(i,j,k) = 0.0; + } else { + pOld_p(i,j,k) = 0.0; + pOld_q(i,j,k) = 0.0; pOld_r(i,j,k) = 0.0; Gam(i,j,k) = 0.0; } - pOld_p(i,j,k) = 0.0; - pOld_q(i,j,k) = 0.0; }); } @@ -130,14 +133,9 @@ void InitializePandRho(Array &P_old, // Calculate charge density from Phi, Nc, Nv, Ec, and Ev - MultiFab acceptor_den(rho.boxArray(), rho.DistributionMap(), 1, 0); - MultiFab donor_den(rho.boxArray(), rho.DistributionMap(), 1, 0); const Array4& hole_den_arr = p_den.array(mfi); const Array4& e_den_arr = e_den.array(mfi); - const Array4& charge_den_arr = rho.array(mfi); - const Array4& acceptor_den_arr = acceptor_den.array(mfi); - const Array4& donor_den_arr = donor_den.array(mfi); const Array4& mask = MaterialMask.array(mfi); @@ -164,18 +162,17 @@ void InitializePandRho(Array &P_old, Real FD_half_p = std::pow(exp(-eta_p) + xi_p, -1.0); hole_den_arr(i,j,k) = 2.0/sqrt(3.14)*Nv*FD_half_p; + } else { - acceptor_den_arr(i,j,k) = acceptor_doping; - donor_den_arr(i,j,k) = 0.0; hole_den_arr(i,j,k) = acceptor_doping; e_den_arr(i,j,k) = intrinsic_carrier_concentration*intrinsic_carrier_concentration/acceptor_doping; - + } - charge_den_arr(i,j,k) = q*(hole_den_arr(i,j,k) - e_den_arr(i,j,k) - acceptor_den_arr(i,j,k) + donor_den_arr(i,j,k)); } else { - charge_den_arr(i,j,k) = 0.; + hole_den_arr(i,j,k) = 0.; + e_den_arr(i,j,k) = 0.; } // //If in channel, set acceptor doping, else (Source/Drain) set donor doping @@ -189,10 +186,52 @@ void InitializePandRho(Array &P_old, }); } - rho.FillBoundary(geom.periodicity()); e_den.FillBoundary(geom.periodicity()); p_den.FillBoundary(geom.periodicity()); + // loop over nodal boxes for rho + for (MFIter mfi(rho); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.validbox(); + + // Calculate charge density from Phi, Nc, Nv, Ec, and Ev + + MultiFab acceptor_den(rho.boxArray(), rho.DistributionMap(), 1, 0); + MultiFab donor_den(rho.boxArray(), rho.DistributionMap(), 1, 0); + + const Array4& hole_den_arr = p_den.array(mfi); + const Array4& e_den_arr = e_den.array(mfi); + const Array4& charge_den_arr = rho.array(mfi); + const Array4& acceptor_den_arr = acceptor_den.array(mfi); + const Array4& donor_den_arr = donor_den.array(mfi); + + const Array4& mask = MaterialMask.array(mfi); + + amrex::ParallelFor( bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + + //SC region + if (mask(i,j,k) >= 2.0) { + + + charge_den_arr(i,j,k) = q*(hole_den_arr(i,j,k) - e_den_arr(i,j,k) - acceptor_den_arr(i,j,k) + donor_den_arr(i,j,k)); + + } else { + charge_den_arr(i,j,k) = 0.; + } + +// //If in channel, set acceptor doping, else (Source/Drain) set donor doping +// if (mask(i,j,k) == 3.0) { +// acceptor_den_arr(i,j,k) = acceptor_doping; +// donor_den_arr(i,j,k) = 0.0; +// } else { // Source / Drain +// acceptor_den_arr(i,j,k) = 0.0; +// donor_den_arr(i,j,k) = donor_doping; +// } + + }); + } + rho.FillBoundary(geom.periodicity()); } // create a mask filled with integers to idetify different material types diff --git a/Source/main.cpp b/Source/main.cpp index e7a836f..6546d02 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -133,23 +133,12 @@ void main_main (c_FerroX& rFerroX) MultiFab e_den(ba, dm, 1, 0); MultiFab charge_den(ba, dm, 1, 0); - //Nodal rho - MultiFab Nodal_hole_den(nba, dm, 1, 0); - MultiFab Nodal_e_den(nba, dm, 1, 0); - MultiFab Nodal_charge_den(nba, dm, 1, 0); - MultiFab MaterialMask(ba, dm, 1, 1); MultiFab tphaseMask(ba, dm, 1, 1); MultiFab angle_alpha(ba, dm, 1, 0); MultiFab angle_beta(ba, dm, 1, 0); MultiFab angle_theta(ba, dm, 1, 0); - //Nodal angles - MultiFab Nodal_MaterialMask(ba, dm, 1, 1); - MultiFab Nodal_angle_alpha(nba, dm, 1, 0); - MultiFab Nodal_angle_beta(nba, dm, 1, 0); - MultiFab Nodal_angle_theta(nba, dm, 1, 0); - for (int dir = 0; dir < AMREX_SPACEDIM; dir++) { P_old[dir].setVal(0.); @@ -170,21 +159,13 @@ void main_main (c_FerroX& rFerroX) Nodal_PoissonPhi.setVal(0.); Nodal_PoissonRHS.setVal(0.); - Nodal_angle_alpha.setVal(0.); - Nodal_angle_beta.setVal(0.); - Nodal_angle_theta.setVal(0.); - Nodal_hole_den.setVal(0.); - Nodal_e_den.setVal(0.); - Nodal_charge_den.setVal(0.); //Initialize material mask - InitializeMaterialMask_Nodal(Nodal_MaterialMask, geom, prob_lo, prob_hi); InitializeMaterialMask(MaterialMask, geom, prob_lo, prob_hi); //InitializeMaterialMask(rFerroX, geom, MaterialMask); if(Coordinate_Transformation == 1){ Initialize_tphase_Mask(rFerroX, geom, tphaseMask); Initialize_Euler_angles(rFerroX, geom, angle_alpha, angle_beta, angle_theta); //cell-centered - Initialize_Euler_angles(rFerroX, geom, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta); //nodal } else { tphaseMask.setVal(0.); } @@ -309,7 +290,7 @@ void main_main (c_FerroX& rFerroX) //Nodal Poisson std::unique_ptr p_mlnode; p_mlnode = std::make_unique(); - p_mlnode->define({geom}, {ba}, {dm}, info); + p_mlnode->define({geom}, {nba}, {dm}, info); //Force singular system to be solvable p_mlnode->setEnforceSingularSolvable(false); @@ -344,7 +325,7 @@ void main_main (c_FerroX& rFerroX) // INITIALIZE P in FE and rho in SC regions //InitializePandRho(P_old, Gamma, charge_den, e_den, hole_den, geom, prob_lo, prob_hi);//old - InitializePandRho(P_old, Gamma, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, Nodal_MaterialMask, tphaseMask, n_cell, geom, prob_lo, prob_hi);//mask based + InitializePandRho(P_old, Gamma, charge_den, e_den, hole_den, MaterialMask, tphaseMask, n_cell, geom, prob_lo, prob_hi);//mask based //Obtain self consisten Phi and rho Real tol = 1.e-5; @@ -369,8 +350,9 @@ void main_main (c_FerroX& rFerroX) pMLMG->apply ({&APoissonPhi_BC}, {&Nodal_PoissonPhi_BC}); //Compute b - ComputePoissonRHS(Nodal_PoissonRHS, P_old, Nodal_charge_den, Nodal_MaterialMask, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta, geom); + ComputePoissonRHS(PoissonRHS, P_old, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + average_cc_to_nodes(Nodal_PoissonRHS, PoissonRHS, geom); //Compute b - A x_H MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); @@ -397,7 +379,8 @@ void main_main (c_FerroX& rFerroX) // Calculate rho from Phi in SC region //ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); //Nodal - ComputeRho(Nodal_PoissonPhi, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, Nodal_MaterialMask); + amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); + ComputeRho(PoissonPhi, charge_den, e_den, hole_den, geom, MaterialMask); if (contains_SC == 0) { // no semiconductor region; set error to zero so the while loop terminates @@ -426,10 +409,6 @@ void main_main (c_FerroX& rFerroX) amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); amrex::average_node_to_cellcenter(PoissonRHS, 0, Nodal_PoissonRHS, 0, 1); - amrex::average_node_to_cellcenter(hole_den, 0, Nodal_hole_den, 0, 1); - amrex::average_node_to_cellcenter(e_den, 0, Nodal_e_den, 0, 1); - amrex::average_node_to_cellcenter(charge_den, 0, Nodal_charge_den, 0, 1); - //amrex::average_node_to_cellcenter(angle_beta, 0, Nodal_angle_beta, 0, 1); // Write a plotfile of the initial data if plot_int > 0 if (plot_int > 0) { @@ -498,7 +477,7 @@ void main_main (c_FerroX& rFerroX) // iterate to compute Phi^{n+1,*} while(err > tol){ - + //Compute x_H SetPhiBC_z(Nodal_PoissonPhi_BC, n_cell); @@ -506,7 +485,9 @@ void main_main (c_FerroX& rFerroX) pMLMG->apply ({&APoissonPhi_BC}, {&Nodal_PoissonPhi_BC}); //Compute b - ComputePoissonRHS(Nodal_PoissonRHS, P_new_pre, Nodal_charge_den, Nodal_MaterialMask, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta, geom); + ComputePoissonRHS(PoissonRHS, P_new_pre, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + + average_cc_to_nodes(Nodal_PoissonRHS, PoissonRHS, geom); //Compute b - A x_H MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); @@ -533,8 +514,9 @@ void main_main (c_FerroX& rFerroX) // Calculate rho from Phi in SC region //ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); - //Nodal - ComputeRho(Nodal_PoissonPhi, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, Nodal_MaterialMask); + //Nodal + amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); + ComputeRho(PoissonPhi, charge_den, e_den, hole_den, geom, MaterialMask); if (contains_SC == 0) { // no semiconductor region; set error to zero so the while loop terminates @@ -590,7 +572,9 @@ void main_main (c_FerroX& rFerroX) pMLMG->apply ({&APoissonPhi_BC}, {&Nodal_PoissonPhi_BC}); //Compute b - ComputePoissonRHS(Nodal_PoissonRHS, P_new, Nodal_charge_den, Nodal_MaterialMask, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta, geom); + ComputePoissonRHS(PoissonRHS, P_new, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + + average_cc_to_nodes(Nodal_PoissonRHS, PoissonRHS, geom); //Compute b - A x_H MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); @@ -619,7 +603,8 @@ void main_main (c_FerroX& rFerroX) // Calculate rho from Phi in SC region //ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); //Nodal - ComputeRho(Nodal_PoissonPhi, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, Nodal_MaterialMask); + amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); + ComputeRho(PoissonPhi, charge_den, e_den, hole_den, geom, MaterialMask); if (contains_SC == 0) { // no semiconductor region; set error to zero so the while loop terminates @@ -701,9 +686,6 @@ void main_main (c_FerroX& rFerroX) amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); amrex::average_node_to_cellcenter(PoissonRHS, 0, Nodal_PoissonRHS, 0, 1); - amrex::average_node_to_cellcenter(hole_den, 0, Nodal_hole_den, 0, 1); - amrex::average_node_to_cellcenter(e_den, 0, Nodal_e_den, 0, 1); - amrex::average_node_to_cellcenter(charge_den, 0, Nodal_charge_den, 0, 1); // Write a plotfile of the current data (plot_int was defined in the inputs file) if (plot_int > 0 && (step%plot_int == 0 || step == steady_state_step)) { @@ -766,7 +748,9 @@ void main_main (c_FerroX& rFerroX) pMLMG->apply ({&APoissonPhi_BC}, {&Nodal_PoissonPhi_BC}); //Compute b - ComputePoissonRHS(Nodal_PoissonRHS, P_old, Nodal_charge_den, Nodal_MaterialMask, Nodal_angle_alpha, Nodal_angle_beta, Nodal_angle_theta, geom); + ComputePoissonRHS(PoissonRHS, P_old, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + + average_cc_to_nodes(Nodal_PoissonRHS, PoissonRHS, geom); //Compute b - A x_H MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); @@ -794,8 +778,9 @@ void main_main (c_FerroX& rFerroX) // Calculate rho from Phi in SC region //ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); - //Nodal - ComputeRho(Nodal_PoissonPhi, Nodal_charge_den, Nodal_e_den, Nodal_hole_den, Nodal_MaterialMask); + //Nodal + amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); + ComputeRho(PoissonPhi, charge_den, e_den, hole_den, geom, MaterialMask); if (contains_SC == 0) { // no semiconductor region; set error to zero so the while loop terminates From aa6db162985e59021f6738a68a23c43652c267c1 Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Tue, 29 Aug 2023 13:47:00 -0700 Subject: [PATCH 12/22] fix rho + nghost for nodal Poisson --- Source/Solver/ChargeDensity.cpp | 4 ++- Source/Solver/ElectrostaticSolver.cpp | 4 ++- Source/Solver/Initialization.cpp | 6 ++-- Source/main.cpp | 44 +++++++++++++-------------- 4 files changed, 32 insertions(+), 26 deletions(-) diff --git a/Source/Solver/ChargeDensity.cpp b/Source/Solver/ChargeDensity.cpp index 49869cd..186f6a1 100644 --- a/Source/Solver/ChargeDensity.cpp +++ b/Source/Solver/ChargeDensity.cpp @@ -87,7 +87,9 @@ void ComputeRho(MultiFab& PoissonPhi, { if (mask(i,j,k) >= 2.0) { - + + acceptor_den_arr(i,j,k) = acceptor_doping; + donor_den_arr(i,j,k) = donor_doping; charge_den_arr(i,j,k) = q*(hole_den_arr(i,j,k) - e_den_arr(i,j,k) - acceptor_den_arr(i,j,k) + donor_den_arr(i,j,k)); } else { diff --git a/Source/Solver/ElectrostaticSolver.cpp b/Source/Solver/ElectrostaticSolver.cpp index 29a4f29..8c724f4 100644 --- a/Source/Solver/ElectrostaticSolver.cpp +++ b/Source/Solver/ElectrostaticSolver.cpp @@ -80,7 +80,8 @@ void ComputePoissonRHS(MultiFab& PoissonRHS, RHS(i,j,k) = charge_den_arr(i,j,k); RHS(i,j,k) *= -1.; - + //amrex::Print() << "RHS(i,j,k) = " << RHS(i,j,k) << "\n"; + //amrex::Print() << "charge_den_arr(i,j,k) = " << charge_den_arr(i,j,k) << "\n"; } else if(mask(i,j,k) == 1.0){ //DE region RHS(i,j,k) = 0.; @@ -95,6 +96,7 @@ void ComputePoissonRHS(MultiFab& PoissonRHS, }); } + PoissonRHS.FillBoundary(geom.periodicity()); } void dF_dPhi(MultiFab& alpha_cc, diff --git a/Source/Solver/Initialization.cpp b/Source/Solver/Initialization.cpp index 9a22d2d..4d54059 100644 --- a/Source/Solver/Initialization.cpp +++ b/Source/Solver/Initialization.cpp @@ -127,7 +127,7 @@ void InitializePandRho(Array &P_old, } // loop over nodal boxes for rho - for (MFIter mfi(rho); mfi.isValid(); ++mfi) + for (MFIter mfi(e_den); mfi.isValid(); ++mfi) { const Box& bx = mfi.validbox(); @@ -213,9 +213,11 @@ void InitializePandRho(Array &P_old, //SC region if (mask(i,j,k) >= 2.0) { - + acceptor_den_arr(i,j,k) = acceptor_doping; + donor_den_arr(i,j,k) = donor_doping; charge_den_arr(i,j,k) = q*(hole_den_arr(i,j,k) - e_den_arr(i,j,k) - acceptor_den_arr(i,j,k) + donor_den_arr(i,j,k)); + //amrex::Print() << "Initialization :: charge_den_arr(i,j,k) = " << charge_den_arr(i,j,k) << "\n"; } else { charge_den_arr(i,j,k) = 0.; } diff --git a/Source/main.cpp b/Source/main.cpp index 6546d02..c4c7298 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -108,7 +108,7 @@ void main_main (c_FerroX& rFerroX) E[dir].define(ba, dm, Ncomp, 0); } - MultiFab PoissonRHS(ba, dm, 1, 0); + MultiFab PoissonRHS(ba, dm, 1, 1); MultiFab PoissonPhi(ba, dm, 1, 1); MultiFab PoissonPhi_Old(ba, dm, 1, 1); MultiFab PoissonPhi_Prev(ba, dm, 1, 1); @@ -125,10 +125,6 @@ void main_main (c_FerroX& rFerroX) MultiFab Nodal_PhiErr(nba, dm, 1, 1); MultiFab Nodal_Phidiff(nba, dm, 1, 1); - MultiFab Ex(ba, dm, 1, 0); - MultiFab Ey(ba, dm, 1, 0); - MultiFab Ez(ba, dm, 1, 0); - MultiFab hole_den(ba, dm, 1, 0); MultiFab e_den(ba, dm, 1, 0); MultiFab charge_den(ba, dm, 1, 0); @@ -151,7 +147,7 @@ void main_main (c_FerroX& rFerroX) } PoissonPhi.setVal(0.); - PoissonRHS.setVal(0.); + PoissonRHS.setVal(0.,1); tphaseMask.setVal(0.); angle_alpha.setVal(0.); angle_beta.setVal(0.); @@ -160,6 +156,10 @@ void main_main (c_FerroX& rFerroX) Nodal_PoissonPhi.setVal(0.); Nodal_PoissonRHS.setVal(0.); + hole_den.setVal(0.); + e_den.setVal(0.); + charge_den.setVal(0.); + //Initialize material mask InitializeMaterialMask(MaterialMask, geom, prob_lo, prob_hi); //InitializeMaterialMask(rFerroX, geom, MaterialMask); @@ -290,7 +290,7 @@ void main_main (c_FerroX& rFerroX) //Nodal Poisson std::unique_ptr p_mlnode; p_mlnode = std::make_unique(); - p_mlnode->define({geom}, {nba}, {dm}, info); + p_mlnode->define({geom}, {ba}, {dm}, info); //Force singular system to be solvable p_mlnode->setEnforceSingularSolvable(false); @@ -352,6 +352,7 @@ void main_main (c_FerroX& rFerroX) //Compute b ComputePoissonRHS(PoissonRHS, P_old, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + //amrex::Abort("compute cell centered Poisson RHS"); average_cc_to_nodes(Nodal_PoissonRHS, PoissonRHS, geom); //Compute b - A x_H MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); @@ -389,13 +390,13 @@ void main_main (c_FerroX& rFerroX) // Calculate Error if (iter > 0){ - MultiFab::Copy(Nodal_PhiErr, Nodal_PoissonPhi, 0, 0, 1, 0); - MultiFab::Subtract(Nodal_PhiErr, Nodal_PoissonPhi_Prev, 0, 0, 1, 0); + MultiFab::Copy(Nodal_PhiErr, Nodal_PoissonPhi, 0, 0, 1, 1); + MultiFab::Subtract(Nodal_PhiErr, Nodal_PoissonPhi_Prev, 0, 0, 1, 1); err = Nodal_PhiErr.norm1(0, geom.periodicity())/Nodal_PoissonPhi.norm1(0, geom.periodicity()); } //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration - MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 0); + MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 1); iter = iter + 1; amrex::Print() << iter << " iterations :: err = " << err << std::endl; @@ -408,7 +409,7 @@ void main_main (c_FerroX& rFerroX) ComputeEfromPhi(Nodal_PoissonPhi, E, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); - amrex::average_node_to_cellcenter(PoissonRHS, 0, Nodal_PoissonRHS, 0, 1); + // Write a plotfile of the initial data if plot_int > 0 if (plot_int > 0) { @@ -525,13 +526,13 @@ void main_main (c_FerroX& rFerroX) // Calculate Error if (iter > 0){ - MultiFab::Copy(Nodal_PhiErr, Nodal_PoissonPhi, 0, 0, 1, 0); - MultiFab::Subtract(Nodal_PhiErr, Nodal_PoissonPhi_Prev, 0, 0, 1, 0); + MultiFab::Copy(Nodal_PhiErr, Nodal_PoissonPhi, 0, 0, 1, 1); + MultiFab::Subtract(Nodal_PhiErr, Nodal_PoissonPhi_Prev, 0, 0, 1, 1); err = Nodal_PhiErr.norm1(0, geom.periodicity())/Nodal_PoissonPhi.norm1(0, geom.periodicity()); } //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration - MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 0); + MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 1); iter = iter + 1; amrex::Print() << iter << " iterations :: err = " << err << std::endl; @@ -613,13 +614,13 @@ void main_main (c_FerroX& rFerroX) // Calculate Error if (iter > 0){ - MultiFab::Copy(Nodal_PhiErr, Nodal_PoissonPhi, 0, 0, 1, 0); - MultiFab::Subtract(Nodal_PhiErr, Nodal_PoissonPhi_Prev, 0, 0, 1, 0); + MultiFab::Copy(Nodal_PhiErr, Nodal_PoissonPhi, 0, 0, 1, 1); + MultiFab::Subtract(Nodal_PhiErr, Nodal_PoissonPhi_Prev, 0, 0, 1, 1); err = Nodal_PhiErr.norm1(0, geom.periodicity())/Nodal_PoissonPhi.norm1(0, geom.periodicity()); } //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration - MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 0); + MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 1); iter = iter + 1; amrex::Print() << iter << " iterations :: err = " << err << std::endl; @@ -665,7 +666,7 @@ void main_main (c_FerroX& rFerroX) } //Copy PoissonPhi to PoissonPhi_Old to calculate difference at the next iteration - MultiFab::Copy(Nodal_PoissonPhi_Old, Nodal_PoissonPhi, 0, 0, 1, 0); + MultiFab::Copy(Nodal_PoissonPhi_Old, Nodal_PoissonPhi, 0, 0, 1, 1); amrex::Print() << "Steady state check : (phi(t) - phi(t-1)).norm0() = " << max_phi_err << std::endl; @@ -685,7 +686,6 @@ void main_main (c_FerroX& rFerroX) amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); - amrex::average_node_to_cellcenter(PoissonRHS, 0, Nodal_PoissonRHS, 0, 1); // Write a plotfile of the current data (plot_int was defined in the inputs file) if (plot_int > 0 && (step%plot_int == 0 || step == steady_state_step)) { @@ -789,13 +789,13 @@ void main_main (c_FerroX& rFerroX) // Calculate Error if (iter > 0){ - MultiFab::Copy(Nodal_PhiErr, Nodal_PoissonPhi, 0, 0, 1, 0); - MultiFab::Subtract(Nodal_PhiErr, Nodal_PoissonPhi_Prev, 0, 0, 1, 0); + MultiFab::Copy(Nodal_PhiErr, Nodal_PoissonPhi, 0, 0, 1, 1); + MultiFab::Subtract(Nodal_PhiErr, Nodal_PoissonPhi_Prev, 0, 0, 1, 1); err = Nodal_PhiErr.norm1(0, geom.periodicity())/Nodal_PoissonPhi.norm1(0, geom.periodicity()); } //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration - MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 0); + MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 1); iter = iter + 1; amrex::Print() << iter << " iterations :: err = " << err << std::endl; From 15ef54a24c3d726d3f3a599fd683ba7edbd5b17d Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Wed, 13 Sep 2023 08:58:57 -0700 Subject: [PATCH 13/22] Newton's method for nodal Poisson --- Source/Solver/ChargeDensity.cpp | 6 ++-- Source/Solver/ElectrostaticSolver.H | 4 ++- Source/Solver/ElectrostaticSolver.cpp | 41 +++++++++++++++++++----- Source/main.cpp | 45 ++++++++++++++++++--------- 4 files changed, 71 insertions(+), 25 deletions(-) diff --git a/Source/Solver/ChargeDensity.cpp b/Source/Solver/ChargeDensity.cpp index 186f6a1..cedf5a2 100644 --- a/Source/Solver/ChargeDensity.cpp +++ b/Source/Solver/ChargeDensity.cpp @@ -72,9 +72,10 @@ void ComputeRho(MultiFab& PoissonPhi, const Box& bx = mfi.validbox(); // Calculate charge density from Phi, Nc, Nv, Ec, and Ev - MultiFab acceptor_den(rho.boxArray(), rho.DistributionMap(), 1, 0); - MultiFab donor_den(rho.boxArray(), rho.DistributionMap(), 1, 0); + MultiFab acceptor_den(rho.boxArray(), rho.DistributionMap(), 1, 1); + MultiFab donor_den(rho.boxArray(), rho.DistributionMap(), 1, 1); + const Array4& phi = PoissonPhi.array(mfi); const Array4& hole_den_arr = p_den.array(mfi); const Array4& e_den_arr = e_den.array(mfi); const Array4& charge_den_arr = rho.array(mfi); @@ -97,6 +98,7 @@ void ComputeRho(MultiFab& PoissonPhi, charge_den_arr(i,j,k) = 0.0; } + //if(std::abs(charge_den_arr(i,j,k) > 1.e20)) amrex::Print() << "ComputeRho :: charge_den_arr(i,j,k) = " << charge_den_arr(i,j,k) << ", phi(i,j,k) = " << phi(i,j,k) << ", e_den_arr = " << e_den_arr(i,j,k) << ", hole_den_arr = " << hole_den_arr(i,j,k) <<"\n"; }); } rho.FillBoundary(geom.periodicity()); diff --git a/Source/Solver/ElectrostaticSolver.H b/Source/Solver/ElectrostaticSolver.H index b6be471..31fef05 100644 --- a/Source/Solver/ElectrostaticSolver.H +++ b/Source/Solver/ElectrostaticSolver.H @@ -53,7 +53,9 @@ void dF_dPhi(MultiFab& alpha_cc, void ComputePoissonRHS_Newton(MultiFab& PoissonRHS, MultiFab& PoissonPhi, - MultiFab& alpha_cc); + MultiFab& PoissonPhi_Prev2, + MultiFab& alpha_cc, + const Geometry& geom); void ComputeAx_H(MultiFab& x_H, MultiFab& Ax_H, const amrex::GpuArray& n_cell); diff --git a/Source/Solver/ElectrostaticSolver.cpp b/Source/Solver/ElectrostaticSolver.cpp index 8c724f4..6a79229 100644 --- a/Source/Solver/ElectrostaticSolver.cpp +++ b/Source/Solver/ElectrostaticSolver.cpp @@ -72,7 +72,8 @@ void ComputePoissonRHS(MultiFab& PoissonRHS, if (R_11 != 1.0 || R_12 != 0.0 || R_13 != 0.0 || R_21 != 0.0 || R_22 != 1.0 || R_23 != 0.0 || R_31 != 0.0 || R_32 != 0.0 || R_33 != 1.0 ){ - amrex::Abort("Coordinate transformation is turned OFF, but rotation matrix is not an identity matrix!"); + amrex::Print() << "alpha = " << alpha_rad << ", beta = " << beta_rad << ", theta = " << theta_rad << "\n"; + amrex::Abort("ComputePoissonRHS : Coordinate transformation is turned OFF, but rotation matrix is not an identity matrix!"); } } @@ -114,11 +115,11 @@ void dF_dPhi(MultiFab& alpha_cc, { - MultiFab PoissonPhi_plus_delta(PoissonPhi.boxArray(), PoissonPhi.DistributionMap(), 1, 0); - MultiFab PoissonRHS_phi_plus_delta(PoissonRHS.boxArray(), PoissonRHS.DistributionMap(), 1, 0); + MultiFab PoissonPhi_plus_delta(PoissonPhi.boxArray(), PoissonPhi.DistributionMap(), 1, 1); + MultiFab PoissonRHS_phi_plus_delta(PoissonRHS.boxArray(), PoissonRHS.DistributionMap(), 1, 1); - MultiFab::Copy(PoissonPhi_plus_delta, PoissonPhi, 0, 0, 1, 0); - PoissonPhi_plus_delta.plus(delta, 0, 1, 0); + MultiFab::Copy(PoissonPhi_plus_delta, PoissonPhi, 0, 0, 1, 1); + PoissonPhi_plus_delta.plus(delta, 0, 1, 1); // Calculate rho from Phi in SC region ComputeRho(PoissonPhi_plus_delta, rho, e_den, p_den, geom, MaterialMask); @@ -126,26 +127,50 @@ void dF_dPhi(MultiFab& alpha_cc, //Compute RHS of Poisson equation ComputePoissonRHS(PoissonRHS_phi_plus_delta, P_old, rho, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); - MultiFab::LinComb(alpha_cc, 1./delta, PoissonRHS_phi_plus_delta, 0, -1./delta, PoissonRHS, 0, 0, 1, 0); + MultiFab::LinComb(alpha_cc, 1./delta, PoissonRHS_phi_plus_delta, 0, -1./delta, PoissonRHS, 0, 0, 1, 1); +// +// for ( MFIter mfi(PoissonPhi); mfi.isValid(); ++mfi ) +// { +// const Box& bx = mfi.validbox(); +// //const Box& bx = mfi.growntilebox(1); +// +// const Array4& alpha = alpha_cc.array(mfi); +// const Array4& phi_p_delta = PoissonPhi_plus_delta.array(mfi); +// const Array4& rhs_phi_p_delta = PoissonRHS_phi_plus_delta.array(mfi); +// const Array4& rhs = PoissonRHS.array(mfi); +// +// amrex::ParallelFor( bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) +// { +// if(std::isnan(alpha(i,j,k))) amrex::Print() <<"alpha(" << i << ", " << j << ", " << k << ") = " << alpha(i,j,k) << ", phi_p_delta = " << phi_p_delta(i,j,k) << ", rhs_phi_p_delta = " << rhs_phi_p_delta(i,j,k) << ", rhs = " << rhs(i,j,k) << "\n"; +// }); +// } } void ComputePoissonRHS_Newton(MultiFab& PoissonRHS, MultiFab& PoissonPhi, - MultiFab& alpha_cc) + MultiFab& PoissonPhi_Prev2, + MultiFab& alpha_cc, + const Geometry& geom) { for ( MFIter mfi(PoissonPhi); mfi.isValid(); ++mfi ) { const Box& bx = mfi.validbox(); + //const Box& bx = mfi.growntilebox(1); const Array4& phi = PoissonPhi.array(mfi); + const Array4& phi_prev2 = PoissonPhi_Prev2.array(mfi); const Array4& poissonRHS = PoissonRHS.array(mfi); const Array4& alpha = alpha_cc.array(mfi); amrex::ParallelFor( bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - poissonRHS(i,j,k) = poissonRHS(i,j,k) - alpha(i,j,k)*phi(i,j,k) ; + //if(std::isnan(alpha(i,j,k))) amrex::Print() << "poissonRHS(i,j,k) = " << poissonRHS(i,j,k) << ", alpha(i,j,k) = " << alpha(i,j,k) << ", phi(i,j,k) = " << phi(i,j,k) << ", phi_prev2(i,j,k) = " << phi_prev2(i,j,k) << "\n"; + amrex::Print() << "poissonRHS(i,j,k) = " << poissonRHS(i,j,k) << ", alpha(i,j,k) = " << alpha(i,j,k) << ", phi(i,j,k) = " << phi(i,j,k) << ", phi_prev2(i,j,k) = " << phi_prev2(i,j,k) << "\n"; + poissonRHS(i,j,k) = poissonRHS(i,j,k) + alpha(i,j,k)*(phi(i,j,k) - phi_prev2(i,j,k)) ; }); } + + PoissonRHS.FillBoundary(geom.periodicity()); } void ComputeEfromPhi(MultiFab& PoissonPhi, diff --git a/Source/main.cpp b/Source/main.cpp index c4c7298..8251ec4 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -112,6 +112,7 @@ void main_main (c_FerroX& rFerroX) MultiFab PoissonPhi(ba, dm, 1, 1); MultiFab PoissonPhi_Old(ba, dm, 1, 1); MultiFab PoissonPhi_Prev(ba, dm, 1, 1); + MultiFab PoissonPhi_Prev2(ba, dm, 1, 1); MultiFab PhiErr(ba, dm, 1, 1); MultiFab Phidiff(ba, dm, 1, 1); @@ -120,20 +121,21 @@ void main_main (c_FerroX& rFerroX) MultiFab Nodal_PoissonPhi(nba, dm, 1, 1); MultiFab Nodal_PoissonPhi_Old(nba, dm, 1, 1); MultiFab Nodal_PoissonPhi_Prev(nba, dm, 1, 1); + MultiFab Nodal_PoissonPhi_Prev2(nba, dm, 1, 1); MultiFab Nodal_PoissonPhi_BC(nba, dm, 1, 0); MultiFab APoissonPhi_BC(nba, dm, 1, 0); MultiFab Nodal_PhiErr(nba, dm, 1, 1); MultiFab Nodal_Phidiff(nba, dm, 1, 1); - MultiFab hole_den(ba, dm, 1, 0); - MultiFab e_den(ba, dm, 1, 0); - MultiFab charge_den(ba, dm, 1, 0); + MultiFab hole_den(ba, dm, 1, 1); + MultiFab e_den(ba, dm, 1, 1); + MultiFab charge_den(ba, dm, 1, 1); MultiFab MaterialMask(ba, dm, 1, 1); MultiFab tphaseMask(ba, dm, 1, 1); - MultiFab angle_alpha(ba, dm, 1, 0); - MultiFab angle_beta(ba, dm, 1, 0); - MultiFab angle_theta(ba, dm, 1, 0); + MultiFab angle_alpha(ba, dm, 1, 1); + MultiFab angle_beta(ba, dm, 1, 1); + MultiFab angle_theta(ba, dm, 1, 1); for (int dir = 0; dir < AMREX_SPACEDIM; dir++) { @@ -147,6 +149,8 @@ void main_main (c_FerroX& rFerroX) } PoissonPhi.setVal(0.); + PoissonPhi_Prev.setVal(0.); + PoissonPhi_Prev2.setVal(0.); PoissonRHS.setVal(0.,1); tphaseMask.setVal(0.); angle_alpha.setVal(0.); @@ -154,6 +158,8 @@ void main_main (c_FerroX& rFerroX) angle_theta.setVal(0.); Nodal_PoissonPhi.setVal(0.); + Nodal_PoissonPhi_Prev.setVal(0.); + Nodal_PoissonPhi_Prev2.setVal(0.); Nodal_PoissonRHS.setVal(0.); hole_den.setVal(0.); @@ -194,7 +200,8 @@ void main_main (c_FerroX& rFerroX) SetPoissonBC(rFerroX, LinOpBCType_2d, all_homogeneous_boundaries, some_functionbased_inhomogeneous_boundaries, some_constant_inhomogeneous_boundaries); // coefficients for solver - MultiFab alpha_cc(ba, dm, 1, 0); + MultiFab alpha_cc(ba, dm, 1, 1); + alpha_cc.setVal(0.); MultiFab beta_cc(ba, dm, 1, 1); std::array< MultiFab, AMREX_SPACEDIM > beta_face; AMREX_D_TERM(beta_face[0].define(convert(ba,IntVect(AMREX_D_DECL(1,0,0))), dm, 1, 0);, @@ -333,6 +340,7 @@ void main_main (c_FerroX& rFerroX) int iter = 0; while(err > tol){ + //while(iter < 2){ /* We would like to solve A x = b with inhomogeneous bc's @@ -352,15 +360,16 @@ void main_main (c_FerroX& rFerroX) //Compute b ComputePoissonRHS(PoissonRHS, P_old, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); - //amrex::Abort("compute cell centered Poisson RHS"); + dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); + + amrex::Print() << "iter = " << iter << "\n"; + ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, PoissonPhi_Prev2, alpha_cc, geom); + average_cc_to_nodes(Nodal_PoissonRHS, PoissonRHS, geom); + //Compute b - A x_H MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); - //dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); - - //ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); - //#ifdef AMREX_USE_EB // p_mlebabec->setACoeffs(0, alpha_cc); //#else @@ -382,7 +391,8 @@ void main_main (c_FerroX& rFerroX) //Nodal amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); ComputeRho(PoissonPhi, charge_den, e_den, hole_den, geom, MaterialMask); - + //if(iter > 0) amrex::Abort("abort after ComputeRho at iter = 1"); + if (contains_SC == 0) { // no semiconductor region; set error to zero so the while loop terminates err = 0.; @@ -395,8 +405,15 @@ void main_main (c_FerroX& rFerroX) err = Nodal_PhiErr.norm1(0, geom.periodicity())/Nodal_PoissonPhi.norm1(0, geom.periodicity()); } - //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration + if (iter == 0) { + MultiFab::Copy(Nodal_PoissonPhi_Prev2, Nodal_PoissonPhi, 0, 0, 1, 1); + } else { + MultiFab::Copy(Nodal_PoissonPhi_Prev2, Nodal_PoissonPhi_Prev, 0, 0, 1, 1); + } + MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 1); + + amrex::average_node_to_cellcenter(PoissonPhi_Prev2, 0, Nodal_PoissonPhi_Prev2, 0, 1); iter = iter + 1; amrex::Print() << iter << " iterations :: err = " << err << std::endl; From 71af586378c1012377ca0027365c3761a3ae0f8c Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Wed, 13 Sep 2023 13:24:44 -0700 Subject: [PATCH 14/22] input file for debugging --- Exec/inputs_newton_debug_mfis | 121 ++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 Exec/inputs_newton_debug_mfis diff --git a/Exec/inputs_newton_debug_mfis b/Exec/inputs_newton_debug_mfis new file mode 100644 index 0000000..03f503c --- /dev/null +++ b/Exec/inputs_newton_debug_mfis @@ -0,0 +1,121 @@ +################################# +###### PROBLEM DOMAIN ###### +################################# + +domain.prob_lo = -8.064e-9 -8.064e-9 0.e-9 +domain.prob_hi = 8.064e-9 8.064e-9 5.8e-9 + +domain.n_cell = 64 64 32 #dx = dy = 5.04A, size of HZO unit cell + +domain.max_grid_size = 64 64 32 +domain.blocking_factor = 64 64 32 + +domain.coord_sys = cartesian + +prob_type = 2 + +TimeIntegratorOrder = 1 + +nsteps = -1 #200 #200000 +plot_int = 500 + +dt = 0.5e-13 + +############################################ +###### COORDINATE TRANSFORMATION ########### +############################################ + +#Coordinate_Transformation = 0 +# +#tphase_geom.tphase_geom_function(x,y,z) = "1. - 1.*(x > -7.5e-9)*(x < 7.5e-9) *(y > -7.5e-9)*(y < 7.5e-9)" +# +##Set different Euler angles in each o-phase in degrees. Note that o-phase is located at (x,y,z) where tphase_geom_function(x,y,z) = 0.0 +##Copy the tphase_geom_function(x,y,z) above and specify the values where the coefficients of the logicals are zero to avoid mistakes. +##It doesn't matter what you set the angles in the t_phase. +#angle_alpha.alpha_function(x,y,z) = "0." +# +#angle_beta.beta_function(x,y,z) = "0." +# +#angle_theta.theta_function(x,y,z) = "0." +# +# +#epsilonX_fe_tphase = 40.0 +# +############################################ +###### POLARIZATION BOUNDARY CONDITIONS #### +############################################ + +P_BC_flag_lo = 3 3 0 +P_BC_flag_hi = 3 3 1 +lambda = 3.0e-10 + +############################################ +###### ELECTRICAL BOUNDARY CONDITIONS ###### +############################################ + +domain.is_periodic = 1 1 0 + +boundary.hi = per per dir(Zmax) +boundary.lo = per per dir(Zmin) + +boundary.Zmax_function = "0.9" +boundary.Zmin_function = "0.0" + +voltage_sweep = 1 +Phi_Bc_lo = 0.0 +Phi_Bc_hi = 0.9 + +Phi_Bc_inc = 0.05 +Phi_Bc_hi_max = 1. +phi_tolerance = 5.e-5 +num_Vapp_max = 20 +#mlmg_verbosity = 2 + +################################# +###### STACK GEOMETRY ########### +################################# + +SC_lo = -8.064e-9 -8.064e-9 0.0e-9 +SC_hi = 8.064e-9 8.064e-9 5.0e-9 + +DE_lo = -8.064e-9 -8.064e-9 5.0e-9 +DE_hi = 8.064e-9 8.064e-9 5.8e-9 + +FE_lo = -1. -1. -1. +FE_hi = -1. -1. -1. + +Channel_lo = -1. -1. -1. +Channel_hi = -1. -1. -1. +#FE_lo = -8.e-9 -8.e-9 4.0e-9 +#FE_hi = 8.e-9 8.e-9 9.0e-9 +##FE:0.0, DE:1.0, Source/Drain:2.0, Channel:3.0 + +#device_geom.device_geom_function(x,y,z) = "0.*(x > -8.e-9)*(x < 8.e-9) * (y > -8.e-9)*(y < 8.e-9) * (z > 4.0e-9)*(z < 9.e-9) +# + 1.*(x > -8.e-9)*(x < 8.e-9) * (y > -8.e-9)*(y < 8.e-9) * (z > 0.0)*(z < 4.e-9)" + +################################# +###### MATERIAL PROPERTIES ###### +################################# + +epsilon_0 = 8.85e-12 +epsilonX_fe = 25.0 +epsilonZ_fe = 25.0 +epsilon_de = 3.9 +epsilon_si = 11.7 +alpha = -2.5e9 +beta = 6.0e10 +gamma = 1.5e11 +BigGamma = 100 +g44 = 1.0e-10 #gy, gz +g11 = 1.0e-10 #-1.0e-12 #gx +#g11 = 1.0e-10 +g44_p = 0.0 +g12 = 0.0 +alpha_12 = 0.0 +alpha_112 = 0.0 +alpha_123 = 0.0 + +acceptor_doping = 1.e21 #9.696e+15 +acceptor_doping = 9.696e+15 +donor_doping = 0. + From 5c42036a70342dbcc77c98ec56ca7afedac9e1cf Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Tue, 26 Sep 2023 16:58:44 -0700 Subject: [PATCH 15/22] use MLNodeABecLaplacian --- Source/Solver/ElectrostaticSolver.H | 5 +- Source/Solver/ElectrostaticSolver.cpp | 17 ++-- Source/main.cpp | 122 ++++++++++++-------------- 3 files changed, 64 insertions(+), 80 deletions(-) diff --git a/Source/Solver/ElectrostaticSolver.H b/Source/Solver/ElectrostaticSolver.H index 31fef05..eb35447 100644 --- a/Source/Solver/ElectrostaticSolver.H +++ b/Source/Solver/ElectrostaticSolver.H @@ -53,9 +53,8 @@ void dF_dPhi(MultiFab& alpha_cc, void ComputePoissonRHS_Newton(MultiFab& PoissonRHS, MultiFab& PoissonPhi, - MultiFab& PoissonPhi_Prev2, - MultiFab& alpha_cc, - const Geometry& geom); + const Geometry& geom, + MultiFab& alpha_cc); void ComputeAx_H(MultiFab& x_H, MultiFab& Ax_H, const amrex::GpuArray& n_cell); diff --git a/Source/Solver/ElectrostaticSolver.cpp b/Source/Solver/ElectrostaticSolver.cpp index 6a79229..b8fcdf7 100644 --- a/Source/Solver/ElectrostaticSolver.cpp +++ b/Source/Solver/ElectrostaticSolver.cpp @@ -80,7 +80,7 @@ void ComputePoissonRHS(MultiFab& PoissonRHS, if(mask(i,j,k) >= 2.0){ //SC region RHS(i,j,k) = charge_den_arr(i,j,k); - RHS(i,j,k) *= -1.; +// RHS(i,j,k) *= -1.; //amrex::Print() << "RHS(i,j,k) = " << RHS(i,j,k) << "\n"; //amrex::Print() << "charge_den_arr(i,j,k) = " << charge_den_arr(i,j,k) << "\n"; } else if(mask(i,j,k) == 1.0){ //DE region @@ -92,7 +92,7 @@ void ComputePoissonRHS(MultiFab& PoissonRHS, - (R_21*NodalDPDx(pOld_q, mask, i, j, k, dx) + R_22*NodalDPDy(pOld_q, mask, i, j, k, dx) + R_23*NodalDPDz(pOld_q, mask, i, j, k, dx)) - (R_31*NodalDPDx(pOld_r, mask, i, j, k, dx) + R_32*NodalDPDy(pOld_r, mask, i, j, k, dx) + R_33*NodalDPDz(pOld_r, mask, i, j, k, dx)); - RHS(i,j,k) *= -1.; + /// RHS(i,j,k) *= -1.; } }); @@ -144,12 +144,12 @@ void dF_dPhi(MultiFab& alpha_cc, // if(std::isnan(alpha(i,j,k))) amrex::Print() <<"alpha(" << i << ", " << j << ", " << k << ") = " << alpha(i,j,k) << ", phi_p_delta = " << phi_p_delta(i,j,k) << ", rhs_phi_p_delta = " << rhs_phi_p_delta(i,j,k) << ", rhs = " << rhs(i,j,k) << "\n"; // }); // } + alpha_cc.FillBoundary(geom.periodicity()); } void ComputePoissonRHS_Newton(MultiFab& PoissonRHS, MultiFab& PoissonPhi, - MultiFab& PoissonPhi_Prev2, - MultiFab& alpha_cc, - const Geometry& geom) + const Geometry& geom, + MultiFab& alpha_cc) { for ( MFIter mfi(PoissonPhi); mfi.isValid(); ++mfi ) @@ -158,15 +158,14 @@ void ComputePoissonRHS_Newton(MultiFab& PoissonRHS, //const Box& bx = mfi.growntilebox(1); const Array4& phi = PoissonPhi.array(mfi); - const Array4& phi_prev2 = PoissonPhi_Prev2.array(mfi); const Array4& poissonRHS = PoissonRHS.array(mfi); const Array4& alpha = alpha_cc.array(mfi); amrex::ParallelFor( bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) { //if(std::isnan(alpha(i,j,k))) amrex::Print() << "poissonRHS(i,j,k) = " << poissonRHS(i,j,k) << ", alpha(i,j,k) = " << alpha(i,j,k) << ", phi(i,j,k) = " << phi(i,j,k) << ", phi_prev2(i,j,k) = " << phi_prev2(i,j,k) << "\n"; - amrex::Print() << "poissonRHS(i,j,k) = " << poissonRHS(i,j,k) << ", alpha(i,j,k) = " << alpha(i,j,k) << ", phi(i,j,k) = " << phi(i,j,k) << ", phi_prev2(i,j,k) = " << phi_prev2(i,j,k) << "\n"; - poissonRHS(i,j,k) = poissonRHS(i,j,k) + alpha(i,j,k)*(phi(i,j,k) - phi_prev2(i,j,k)) ; + //amrex::Print() << "poissonRHS(i,j,k) = " << poissonRHS(i,j,k) << ", alpha(i,j,k) = " << alpha(i,j,k) << ", phi(i,j,k) = " << phi(i,j,k) << ", phi_prev2(i,j,k) = " << phi_prev2(i,j,k) << "\n"; + poissonRHS(i,j,k) = poissonRHS(i,j,k) - alpha(i,j,k)*phi(i,j,k) ; }); } @@ -565,7 +564,7 @@ void Fill_FunctionBased_Inhomogeneous_Boundaries(c_FerroX& rFerroX, MultiFab& Po //A multifab filled with zeros, but boundary cells filled to respect bc's void SetPhiBC_z(MultiFab& PoissonPhi, const amrex::GpuArray& n_cell) { - PoissonPhi.setVal(0.); +// PoissonPhi.setVal(0.); for (MFIter mfi(PoissonPhi); mfi.isValid(); ++mfi) { const Box& bx = mfi.validbox(); diff --git a/Source/main.cpp b/Source/main.cpp index 8251ec4..d2f3724 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #ifdef AMREX_USE_EB #include @@ -151,7 +152,8 @@ void main_main (c_FerroX& rFerroX) PoissonPhi.setVal(0.); PoissonPhi_Prev.setVal(0.); PoissonPhi_Prev2.setVal(0.); - PoissonRHS.setVal(0.,1); + PoissonRHS.setVal(0.); + Phidiff.setVal(0.); tphaseMask.setVal(0.); angle_alpha.setVal(0.); angle_beta.setVal(0.); @@ -192,9 +194,9 @@ void main_main (c_FerroX& rFerroX) amrex::Print() << "contains_SC = " << contains_SC << "\n"; #ifdef AMREX_USE_EB - MultiFab Plt(ba, dm, 18, 0, MFInfo(), *rGprop.pEB->p_factory_union); + MultiFab Plt(ba, dm, 19, 0, MFInfo(), *rGprop.pEB->p_factory_union); #else - MultiFab Plt(ba, dm, 18, 0); + MultiFab Plt(ba, dm, 19, 0); #endif SetPoissonBC(rFerroX, LinOpBCType_2d, all_homogeneous_boundaries, some_functionbased_inhomogeneous_boundaries, some_constant_inhomogeneous_boundaries); @@ -202,6 +204,10 @@ void main_main (c_FerroX& rFerroX) // coefficients for solver MultiFab alpha_cc(ba, dm, 1, 1); alpha_cc.setVal(0.); + + MultiFab alpha_nd(nba, dm, 1, 0); + alpha_nd.setVal(0.); + MultiFab beta_cc(ba, dm, 1, 1); std::array< MultiFab, AMREX_SPACEDIM > beta_face; AMREX_D_TERM(beta_face[0].define(convert(ba,IntVect(AMREX_D_DECL(1,0,0))), dm, 1, 0);, @@ -295,35 +301,43 @@ void main_main (c_FerroX& rFerroX) // pMLMG->setVerbose(mlmg_verbosity); //Nodal Poisson - std::unique_ptr p_mlnode; - p_mlnode = std::make_unique(); - p_mlnode->define({geom}, {ba}, {dm}, info); + std::unique_ptr p_mlndabec; + p_mlndabec = std::make_unique(); + p_mlndabec->define({geom}, {ba}, {dm}, info); //Force singular system to be solvable - p_mlnode->setEnforceSingularSolvable(false); + p_mlndabec->setEnforceSingularSolvable(false); - p_mlnode->setMaxOrder(linop_maxorder); + p_mlndabec->setMaxOrder(linop_maxorder); - p_mlnode->setDomainBC(LinOpBCType_2d[0], LinOpBCType_2d[1]); + p_mlndabec->setDomainBC(LinOpBCType_2d[0], LinOpBCType_2d[1]); - if(some_constant_inhomogeneous_boundaries) - { - Fill_Constant_Inhomogeneous_Boundaries(rFerroX, Nodal_PoissonPhi); - } - if(some_functionbased_inhomogeneous_boundaries) - { - Fill_FunctionBased_Inhomogeneous_Boundaries(rFerroX, Nodal_PoissonPhi, time); - } - Nodal_PoissonPhi.FillBoundary(geom.periodicity()); + //if(some_constant_inhomogeneous_boundaries) + //{ + // Fill_Constant_Inhomogeneous_Boundaries(rFerroX, Nodal_PoissonPhi); + //} + //if(some_functionbased_inhomogeneous_boundaries) + //{ + // Fill_FunctionBased_Inhomogeneous_Boundaries(rFerroX, Nodal_PoissonPhi, time); + //} + //Nodal_PoissonPhi.FillBoundary(geom.periodicity()); + //SetPhiBC_z(Nodal_PoissonPhi, n_cell); + // set Dirichlet BC by reading in the ghost cell values - //p_mlnode->setLevelBC(amrlev, &Nodal_PoissonPhi); + //p_mlndabec->setLevelBC(amrlev, &Nodal_PoissonPhi); // (div beta grad) phi = rhs - p_mlnode->setSigma(amrlev, beta_cc); + //p_mlnode->setSigma(amrlev, beta_cc); + + // (A*alpha_cc - B * div beta grad) phi = rhs where A, phi and rhs are nodal MultiFabs and B is cell-centered. + p_mlndabec->setScalars(-1.0, -1.0); // A = -1.0, B = 1.0; solving (-alpha - div beta grad) phi = RHS + //p_mlndabec->setBCoeffs(amrlev, amrex::GetArrOfConstPtrs(beta_face)); + p_mlndabec->setBCoeffs(amrlev, beta_cc); + //Declare MLMG object - pMLMG = std::make_unique(*p_mlnode); + pMLMG = std::make_unique(*p_mlndabec); pMLMG->setVerbose(mlmg_verbosity); #endif @@ -331,7 +345,6 @@ void main_main (c_FerroX& rFerroX) // INITIALIZE P in FE and rho in SC regions - //InitializePandRho(P_old, Gamma, charge_den, e_den, hole_den, geom, prob_lo, prob_hi);//old InitializePandRho(P_old, Gamma, charge_den, e_den, hole_den, MaterialMask, tphaseMask, n_cell, geom, prob_lo, prob_hi);//mask based //Obtain self consisten Phi and rho @@ -339,59 +352,39 @@ void main_main (c_FerroX& rFerroX) Real err = 1.0; int iter = 0; - while(err > tol){ - //while(iter < 2){ - -/* - We would like to solve A x = b with inhomogeneous bc's - Here, "A" is div sigma grad - This is equivalent to A_H x = b - A x_H, where - A is the inhomogeneous operator - A_H is the homogeneous operator - x_H is a multifab filled with zeros, but boundary cells filled to respect bc's -*/ - - //Compute x_H - SetPhiBC_z(Nodal_PoissonPhi_BC, n_cell); - - //Compute A x_H - pMLMG->apply ({&APoissonPhi_BC}, {&Nodal_PoissonPhi_BC}); - - //Compute b + //while(err > tol){ + while(iter < 2){ + ComputePoissonRHS(PoissonRHS, P_old, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); amrex::Print() << "iter = " << iter << "\n"; - ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, PoissonPhi_Prev2, alpha_cc, geom); + ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, geom, alpha_cc); average_cc_to_nodes(Nodal_PoissonRHS, PoissonRHS, geom); - - //Compute b - A x_H - MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); -//#ifdef AMREX_USE_EB -// p_mlebabec->setACoeffs(0, alpha_cc); -//#else -// p_mlabec->setACoeffs(0, alpha_cc); -//#endif + average_cc_to_nodes(alpha_nd, alpha_cc, geom); + +#ifdef AMREX_USE_EB + p_mlebabec->setACoeffs(0, alpha_cc); +#else + p_mlndabec->setACoeffs(0, alpha_nd); +#endif + //Initial guess for phi Nodal_PoissonPhi.setVal(0.); + SetPhiBC_z(Nodal_PoissonPhi, n_cell); //Poisson Solve pMLMG->solve({&Nodal_PoissonPhi}, {&Nodal_PoissonRHS}, 1.e-10, -1); - //Fill inhomogenous Dirichlet BC in boundary nodes after the solve - SetPhiBC_z_after_solve(Nodal_PoissonPhi, n_cell); - Nodal_PoissonPhi.FillBoundary(geom.periodicity()); - // Calculate rho from Phi in SC region - //ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); - //Nodal amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); + + // Calculate rho from Phi in SC region ComputeRho(PoissonPhi, charge_den, e_den, hole_den, geom, MaterialMask); - //if(iter > 0) amrex::Abort("abort after ComputeRho at iter = 1"); if (contains_SC == 0) { // no semiconductor region; set error to zero so the while loop terminates @@ -405,16 +398,8 @@ void main_main (c_FerroX& rFerroX) err = Nodal_PhiErr.norm1(0, geom.periodicity())/Nodal_PoissonPhi.norm1(0, geom.periodicity()); } - if (iter == 0) { - MultiFab::Copy(Nodal_PoissonPhi_Prev2, Nodal_PoissonPhi, 0, 0, 1, 1); - } else { - MultiFab::Copy(Nodal_PoissonPhi_Prev2, Nodal_PoissonPhi_Prev, 0, 0, 1, 1); - } - MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 1); - amrex::average_node_to_cellcenter(PoissonPhi_Prev2, 0, Nodal_PoissonPhi_Prev2, 0, 1); - iter = iter + 1; amrex::Print() << iter << " iterations :: err = " << err << std::endl; } @@ -425,7 +410,7 @@ void main_main (c_FerroX& rFerroX) // Calculate E from Phi ComputeEfromPhi(Nodal_PoissonPhi, E, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); - amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); + //amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); // Write a plotfile of the initial data if plot_int > 0 if (plot_int > 0) @@ -451,10 +436,11 @@ void main_main (c_FerroX& rFerroX) MultiFab::Copy(Plt, angle_beta, 0, 15, 1, 0); MultiFab::Copy(Plt, angle_theta, 0, 16, 1, 0); MultiFab::Copy(Plt, Phidiff, 0, 17, 1, 0); + MultiFab::Copy(Plt, alpha_cc, 0, 18, 1, 0); #ifdef AMREX_USE_EB - amrex::EB_WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "tphase","alpha", "beta", "theta", "PhiDiff"}, geom, time, step); + amrex::EB_WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "tphase","alpha", "beta", "theta", "PhiDiff", "alpha_cc"}, geom, time, step); #else - amrex::WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "tphase","alpha", "beta", "theta", "PhiDiff"}, geom, time, step); + amrex::WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "tphase","alpha", "beta", "theta", "PhiDiff", "alpha_cc"}, geom, time, step); #endif } From 1e05d762c8d9ba24f08f58c5d5dd9389d312c926 Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Wed, 27 Sep 2023 13:50:32 -0700 Subject: [PATCH 16/22] some fixes for nodelAbecLap --- Source/Solver/ElectrostaticSolver.cpp | 9 ++++++--- Source/main.cpp | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Source/Solver/ElectrostaticSolver.cpp b/Source/Solver/ElectrostaticSolver.cpp index b8fcdf7..194ec84 100644 --- a/Source/Solver/ElectrostaticSolver.cpp +++ b/Source/Solver/ElectrostaticSolver.cpp @@ -88,9 +88,12 @@ void ComputePoissonRHS(MultiFab& PoissonRHS, RHS(i,j,k) = 0.; } else { //mask(i,j,k) == 0.0 FE region - RHS(i,j,k) = - (R_11*NodalDPDx(pOld_p, mask, i, j, k, dx) + R_12*NodalDPDy(pOld_p, mask, i, j, k, dx) + R_13*NodalDPDz(pOld_p, mask, i, j, k, dx)) - - (R_21*NodalDPDx(pOld_q, mask, i, j, k, dx) + R_22*NodalDPDy(pOld_q, mask, i, j, k, dx) + R_23*NodalDPDz(pOld_q, mask, i, j, k, dx)) - - (R_31*NodalDPDx(pOld_r, mask, i, j, k, dx) + R_32*NodalDPDy(pOld_r, mask, i, j, k, dx) + R_33*NodalDPDz(pOld_r, mask, i, j, k, dx)); + //RHS(i,j,k) = - (R_11*NodalDPDx(pOld_p, mask, i, j, k, dx) + R_12*NodalDPDy(pOld_p, mask, i, j, k, dx) + R_13*NodalDPDz(pOld_p, mask, i, j, k, dx)) + // - (R_21*NodalDPDx(pOld_q, mask, i, j, k, dx) + R_22*NodalDPDy(pOld_q, mask, i, j, k, dx) + R_23*NodalDPDz(pOld_q, mask, i, j, k, dx)) + // - (R_31*NodalDPDx(pOld_r, mask, i, j, k, dx) + R_32*NodalDPDy(pOld_r, mask, i, j, k, dx) + R_33*NodalDPDz(pOld_r, mask, i, j, k, dx)); + RHS(i,j,k) = - (R_11*DPDx(pOld_p, mask, i, j, k, dx) + R_12*DPDy(pOld_p, mask, i, j, k, dx) + R_13*DPDz(pOld_p, mask, i, j, k, dx)) + - (R_21*DPDx(pOld_q, mask, i, j, k, dx) + R_22*DPDy(pOld_q, mask, i, j, k, dx) + R_23*DPDz(pOld_q, mask, i, j, k, dx)) + - (R_31*DPDx(pOld_r, mask, i, j, k, dx) + R_32*DPDy(pOld_r, mask, i, j, k, dx) + R_33*DPDz(pOld_r, mask, i, j, k, dx)); /// RHS(i,j,k) *= -1.; } diff --git a/Source/main.cpp b/Source/main.cpp index d2f3724..1e49eb3 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -331,7 +331,7 @@ void main_main (c_FerroX& rFerroX) //p_mlnode->setSigma(amrlev, beta_cc); // (A*alpha_cc - B * div beta grad) phi = rhs where A, phi and rhs are nodal MultiFabs and B is cell-centered. - p_mlndabec->setScalars(-1.0, -1.0); // A = -1.0, B = 1.0; solving (-alpha - div beta grad) phi = RHS + p_mlndabec->setScalars(-1.0, 1.0); // A = -1.0, B = 1.0; solving (-alpha - div beta grad) phi = RHS //p_mlndabec->setBCoeffs(amrlev, amrex::GetArrOfConstPtrs(beta_face)); p_mlndabec->setBCoeffs(amrlev, beta_cc); From aa2fb869c6116027ddddda56d956517bd24f9212 Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Wed, 27 Sep 2023 15:30:15 -0700 Subject: [PATCH 17/22] boundary points fixes to average_cc_to_nodes --- Source/Solver/ElectrostaticSolver.cpp | 13 +++++++++++++ Source/main.cpp | 6 ++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Source/Solver/ElectrostaticSolver.cpp b/Source/Solver/ElectrostaticSolver.cpp index 194ec84..02e7e60 100644 --- a/Source/Solver/ElectrostaticSolver.cpp +++ b/Source/Solver/ElectrostaticSolver.cpp @@ -626,7 +626,20 @@ void average_cc_to_nodes(MultiFab& mf_nodal, const MultiFab& mf_cc, const Geomet + mf_cc_arr(i, j-1, k ) + mf_cc_arr(i-1, j, k ) + mf_cc_arr(i, j, k )); + if(k == 0){ + mf_nodal_arr(i,j,k) = 1./4.*(mf_cc_arr(i-1, j-1, k ) + + mf_cc_arr(i, j-1, k ) + + mf_cc_arr(i-1, j, k ) + + mf_cc_arr(i, j, k )); + } + if(k == 32){ + mf_nodal_arr(i,j,k) = 1./4.*(mf_cc_arr(i-1, j-1,k-1 ) + + mf_cc_arr(i, j-1, k-1 ) + + mf_cc_arr(i-1, j, k-1 ) + + mf_cc_arr(i, j, k-1 )); + } }); + } mf_nodal.FillBoundary(geom.periodicity()); } diff --git a/Source/main.cpp b/Source/main.cpp index 1e49eb3..5c37322 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -352,8 +352,7 @@ void main_main (c_FerroX& rFerroX) Real err = 1.0; int iter = 0; - //while(err > tol){ - while(iter < 2){ + while(err > tol){ ComputePoissonRHS(PoissonRHS, P_old, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); @@ -366,6 +365,9 @@ void main_main (c_FerroX& rFerroX) average_cc_to_nodes(alpha_nd, alpha_cc, geom); + VisMF::Write(PoissonRHS, "poissonrhs"); + VisMF::Write(alpha_cc, "alpha_cc"); + #ifdef AMREX_USE_EB p_mlebabec->setACoeffs(0, alpha_cc); #else From 6c32e91ee983ce5b6849430d9c4d75095be533be Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Tue, 3 Oct 2023 10:18:16 -0700 Subject: [PATCH 18/22] debug: output at every iter --- Source/main.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Source/main.cpp b/Source/main.cpp index 5c37322..e9e2927 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -405,7 +405,6 @@ void main_main (c_FerroX& rFerroX) iter = iter + 1; amrex::Print() << iter << " iterations :: err = " << err << std::endl; } - } amrex::Print() << "\n ========= Self-Consistent Initialization of P and Rho Done! ========== \n"<< iter << " iterations to obtain self consistent Phi with err = " << err << std::endl; @@ -417,7 +416,8 @@ void main_main (c_FerroX& rFerroX) // Write a plotfile of the initial data if plot_int > 0 if (plot_int > 0) { - int step = 0; + //int step = 0; + int step = iter; const std::string& pltfile = amrex::Concatenate("plt",step,8); MultiFab::Copy(Plt, P_old[0], 0, 0, 1, 0); MultiFab::Copy(Plt, P_old[1], 0, 1, 1, 0); @@ -446,6 +446,7 @@ void main_main (c_FerroX& rFerroX) #endif } + } amrex::Print() << "\n ========= Advance Steps ========== \n"<< std::endl; int steady_state_step = 1000000; //Initialize to a large number. It will be overwritten by the time step at which steady state condition is satidfied From d8496af23d5098e44ff3b7a0b0214e584f1c0cdc Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Tue, 3 Oct 2023 11:43:45 -0700 Subject: [PATCH 19/22] make initialization and rho computation same as dev --- Source/Solver/ChargeDensity.cpp | 61 +++++------------- Source/Solver/Initialization.cpp | 102 ++++++++----------------------- 2 files changed, 41 insertions(+), 122 deletions(-) diff --git a/Source/Solver/ChargeDensity.cpp b/Source/Solver/ChargeDensity.cpp index cedf5a2..21ad682 100644 --- a/Source/Solver/ChargeDensity.cpp +++ b/Source/Solver/ChargeDensity.cpp @@ -14,10 +14,15 @@ void ComputeRho(MultiFab& PoissonPhi, const Box& bx = mfi.validbox(); // Calculate charge density from Phi, Nc, Nv, Ec, and Ev + MultiFab acceptor_den(rho.boxArray(), rho.DistributionMap(), 1, 1); + MultiFab donor_den(rho.boxArray(), rho.DistributionMap(), 1, 1); const Array4& hole_den_arr = p_den.array(mfi); const Array4& e_den_arr = e_den.array(mfi); + const Array4& charge_den_arr = rho.array(mfi); const Array4& phi = PoissonPhi.array(mfi); + const Array4& acceptor_den_arr = acceptor_den.array(mfi); + const Array4& donor_den_arr = donor_den.array(mfi); const Array4& mask = MaterialMask.array(mfi); @@ -43,54 +48,21 @@ void ComputeRho(MultiFab& PoissonPhi, hole_den_arr(i,j,k) = 2.0/sqrt(3.14)*Nv*FD_half_p; } else { //Maxwell-Boltzmann - Real p_0 = acceptor_doping; - Real n_0 = intrinsic_carrier_concentration*intrinsic_carrier_concentration/p_0; + Real n_0 = intrinsic_carrier_concentration; + Real p_0 = intrinsic_carrier_concentration; hole_den_arr(i,j,k) = p_0*exp(-(q*phi(i,j,k))/(kb*T)); e_den_arr(i,j,k) = n_0*exp(q*phi(i,j,k)/(kb*T)); } - ////If in channel, set acceptor doping, else (Source/Drain) set donor doping - //if (mask(i,j,k) == 3.0) { - // acceptor_den_arr(i,j,k) = acceptor_doping; - // donor_den_arr(i,j,k) = 0.0; - //} else { // Source / Drain - // acceptor_den_arr(i,j,k) = 0.0; - // donor_den_arr(i,j,k) = donor_doping; - //} - - - } else { - hole_den_arr(i,j,k) = 0.; - e_den_arr(i,j,k) = 0.; - } - }); - } - - // loop over boxes - for (MFIter mfi(PoissonPhi); mfi.isValid(); ++mfi) - { - const Box& bx = mfi.validbox(); - - // Calculate charge density from Phi, Nc, Nv, Ec, and Ev - MultiFab acceptor_den(rho.boxArray(), rho.DistributionMap(), 1, 1); - MultiFab donor_den(rho.boxArray(), rho.DistributionMap(), 1, 1); - - const Array4& phi = PoissonPhi.array(mfi); - const Array4& hole_den_arr = p_den.array(mfi); - const Array4& e_den_arr = e_den.array(mfi); - const Array4& charge_den_arr = rho.array(mfi); - const Array4& acceptor_den_arr = acceptor_den.array(mfi); - const Array4& donor_den_arr = donor_den.array(mfi); - const Array4& mask = MaterialMask.array(mfi); - - - amrex::ParallelFor( bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) - { + //If in channel, set acceptor doping, else (Source/Drain) set donor doping + if (mask(i,j,k) == 3.0) { + acceptor_den_arr(i,j,k) = acceptor_doping; + donor_den_arr(i,j,k) = 0.0; + } else { // Source / Drain + acceptor_den_arr(i,j,k) = 0.0; + donor_den_arr(i,j,k) = donor_doping; + } - if (mask(i,j,k) >= 2.0) { - - acceptor_den_arr(i,j,k) = acceptor_doping; - donor_den_arr(i,j,k) = donor_doping; charge_den_arr(i,j,k) = q*(hole_den_arr(i,j,k) - e_den_arr(i,j,k) - acceptor_den_arr(i,j,k) + donor_den_arr(i,j,k)); } else { @@ -98,8 +70,7 @@ void ComputeRho(MultiFab& PoissonPhi, charge_den_arr(i,j,k) = 0.0; } - //if(std::abs(charge_den_arr(i,j,k) > 1.e20)) amrex::Print() << "ComputeRho :: charge_den_arr(i,j,k) = " << charge_den_arr(i,j,k) << ", phi(i,j,k) = " << phi(i,j,k) << ", e_den_arr = " << e_den_arr(i,j,k) << ", hole_den_arr = " << hole_den_arr(i,j,k) <<"\n"; }); } rho.FillBoundary(geom.periodicity()); - } + } diff --git a/Source/Solver/Initialization.cpp b/Source/Solver/Initialization.cpp index 4d54059..5834e89 100644 --- a/Source/Solver/Initialization.cpp +++ b/Source/Solver/Initialization.cpp @@ -119,25 +119,18 @@ void InitializePandRho(Array &P_old, Gam(i,j,k) = 0.0; } }); - } - - for (int i = 0; i < 3; i++){ - // fill periodic ghost cells - P_old[i].FillBoundary(geom.periodicity()); - } - - // loop over nodal boxes for rho - for (MFIter mfi(e_den); mfi.isValid(); ++mfi) - { - const Box& bx = mfi.validbox(); - - // Calculate charge density from Phi, Nc, Nv, Ec, and Ev + + // Calculate charge density from Phi, Nc, Nv, Ec, and Ev + MultiFab acceptor_den(rho.boxArray(), rho.DistributionMap(), 1, 1); + MultiFab donor_den(rho.boxArray(), rho.DistributionMap(), 1, 1); const Array4& hole_den_arr = p_den.array(mfi); const Array4& e_den_arr = e_den.array(mfi); + const Array4& charge_den_arr = rho.array(mfi); + const Array4& acceptor_den_arr = acceptor_den.array(mfi); + const Array4& donor_den_arr = donor_den.array(mfi); - const Array4& mask = MaterialMask.array(mfi); amrex::ParallelFor( bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) { @@ -162,77 +155,32 @@ void InitializePandRho(Array &P_old, Real FD_half_p = std::pow(exp(-eta_p) + xi_p, -1.0); hole_den_arr(i,j,k) = 2.0/sqrt(3.14)*Nv*FD_half_p; - } else { - hole_den_arr(i,j,k) = acceptor_doping; - e_den_arr(i,j,k) = intrinsic_carrier_concentration*intrinsic_carrier_concentration/acceptor_doping; - - } - } else { - hole_den_arr(i,j,k) = 0.; - e_den_arr(i,j,k) = 0.; - } + hole_den_arr(i,j,k) = intrinsic_carrier_concentration; + e_den_arr(i,j,k) = intrinsic_carrier_concentration; -// //If in channel, set acceptor doping, else (Source/Drain) set donor doping -// if (mask(i,j,k) == 3.0) { -// acceptor_den_arr(i,j,k) = acceptor_doping; -// donor_den_arr(i,j,k) = 0.0; -// } else { // Source / Drain -// acceptor_den_arr(i,j,k) = 0.0; -// donor_den_arr(i,j,k) = donor_doping; -// } + } + } + + //If in channel, set acceptor doping, else (Source/Drain) set donor doping + if (mask(i,j,k) == 3.0) { + acceptor_den_arr(i,j,k) = acceptor_doping; + donor_den_arr(i,j,k) = 0.0; + } else { // Source / Drain + acceptor_den_arr(i,j,k) = 0.0; + donor_den_arr(i,j,k) = donor_doping; + } + charge_den_arr(i,j,k) = q*(hole_den_arr(i,j,k) - e_den_arr(i,j,k) - acceptor_den_arr(i,j,k) + donor_den_arr(i,j,k)); }); } - e_den.FillBoundary(geom.periodicity()); - p_den.FillBoundary(geom.periodicity()); - - // loop over nodal boxes for rho - for (MFIter mfi(rho); mfi.isValid(); ++mfi) - { - const Box& bx = mfi.validbox(); - - // Calculate charge density from Phi, Nc, Nv, Ec, and Ev - - MultiFab acceptor_den(rho.boxArray(), rho.DistributionMap(), 1, 0); - MultiFab donor_den(rho.boxArray(), rho.DistributionMap(), 1, 0); - - const Array4& hole_den_arr = p_den.array(mfi); - const Array4& e_den_arr = e_den.array(mfi); - const Array4& charge_den_arr = rho.array(mfi); - const Array4& acceptor_den_arr = acceptor_den.array(mfi); - const Array4& donor_den_arr = donor_den.array(mfi); - - const Array4& mask = MaterialMask.array(mfi); - - amrex::ParallelFor( bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) - { - - //SC region - if (mask(i,j,k) >= 2.0) { - - acceptor_den_arr(i,j,k) = acceptor_doping; - donor_den_arr(i,j,k) = donor_doping; - charge_den_arr(i,j,k) = q*(hole_den_arr(i,j,k) - e_den_arr(i,j,k) - acceptor_den_arr(i,j,k) + donor_den_arr(i,j,k)); - - //amrex::Print() << "Initialization :: charge_den_arr(i,j,k) = " << charge_den_arr(i,j,k) << "\n"; - } else { - charge_den_arr(i,j,k) = 0.; - } - -// //If in channel, set acceptor doping, else (Source/Drain) set donor doping -// if (mask(i,j,k) == 3.0) { -// acceptor_den_arr(i,j,k) = acceptor_doping; -// donor_den_arr(i,j,k) = 0.0; -// } else { // Source / Drain -// acceptor_den_arr(i,j,k) = 0.0; -// donor_den_arr(i,j,k) = donor_doping; -// } - - }); + for (int i = 0; i < 3; i++){ + // fill periodic ghost cells + P_old[i].FillBoundary(geom.periodicity()); } + rho.FillBoundary(geom.periodicity()); } From eb86790f80f712aef1110f5d6ccb6c7d70f2bca7 Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Tue, 3 Oct 2023 13:57:05 -0700 Subject: [PATCH 20/22] try OverrideSync --- Source/main.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/main.cpp b/Source/main.cpp index e9e2927..5f5821e 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -365,9 +365,9 @@ void main_main (c_FerroX& rFerroX) average_cc_to_nodes(alpha_nd, alpha_cc, geom); - VisMF::Write(PoissonRHS, "poissonrhs"); - VisMF::Write(alpha_cc, "alpha_cc"); - + Nodal_PoissonRHS.OverrideSync(geom.periodicity()); + alpha_nd.OverrideSync(geom.periodicity()); + #ifdef AMREX_USE_EB p_mlebabec->setACoeffs(0, alpha_cc); #else From 6e05764292dfbf16b1421b36ee5fbe5af944b213 Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Tue, 3 Oct 2023 20:50:44 -0700 Subject: [PATCH 21/22] nghost = 1 in LinComb to calculate alpfa_cc --- Source/Solver/ElectrostaticSolver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Solver/ElectrostaticSolver.cpp b/Source/Solver/ElectrostaticSolver.cpp index 02e7e60..2f40bad 100644 --- a/Source/Solver/ElectrostaticSolver.cpp +++ b/Source/Solver/ElectrostaticSolver.cpp @@ -130,7 +130,7 @@ void dF_dPhi(MultiFab& alpha_cc, //Compute RHS of Poisson equation ComputePoissonRHS(PoissonRHS_phi_plus_delta, P_old, rho, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); - MultiFab::LinComb(alpha_cc, 1./delta, PoissonRHS_phi_plus_delta, 0, -1./delta, PoissonRHS, 0, 0, 1, 1); + MultiFab::LinComb(alpha_cc, 1./delta, PoissonRHS_phi_plus_delta, 0, -1./delta, PoissonRHS, 0, 0, 1, 0); // // for ( MFIter mfi(PoissonPhi); mfi.isValid(); ++mfi ) // { From 56fb0730940eb500485955f33329f2ed6a0411f0 Mon Sep 17 00:00:00 2001 From: Prabhat Kumar Date: Wed, 4 Oct 2023 10:21:54 -0700 Subject: [PATCH 22/22] bringing some moe stuff from latest dev manually --- Source/FerroX.cpp | 4 + Source/FerroX_namespace.H | 1 + Source/Solver/ChargeDensity.cpp | 6 +- Source/Solver/Initialization.cpp | 8 +- Source/main.cpp | 348 ++++++++++++++----------------- 5 files changed, 174 insertions(+), 193 deletions(-) diff --git a/Source/FerroX.cpp b/Source/FerroX.cpp index 57618a4..886e47d 100644 --- a/Source/FerroX.cpp +++ b/Source/FerroX.cpp @@ -237,6 +237,7 @@ AMREX_GPU_MANAGED amrex::Real FerroX::Phi_Bc_inc; AMREX_GPU_MANAGED amrex::Real FerroX::Phi_Bc_hi_max; AMREX_GPU_MANAGED amrex::Real FerroX::phi_tolerance; AMREX_GPU_MANAGED int FerroX::random_seed; +AMREX_GPU_MANAGED int FerroX::num_Vapp_max; //Maximum number of applied voltage points to sweep void InitializeFerroXNamespace(const amrex::GpuArray& prob_lo, const amrex::GpuArray& prob_hi) { @@ -345,6 +346,9 @@ void InitializeFerroXNamespace(const amrex::GpuArray &P_old, } else { - hole_den_arr(i,j,k) = intrinsic_carrier_concentration; - e_den_arr(i,j,k) = intrinsic_carrier_concentration; - + //hole_den_arr(i,j,k) = intrinsic_carrier_concentration; + //e_den_arr(i,j,k) = intrinsic_carrier_concentration; + hole_den_arr(i,j,k) = acceptor_doping; + e_den_arr(i,j,k) = intrinsic_carrier_concentration*intrinsic_carrier_concentration/acceptor_doping; + } } diff --git a/Source/main.cpp b/Source/main.cpp index 5f5821e..ad86a0f 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -358,16 +358,12 @@ void main_main (c_FerroX& rFerroX) dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); - amrex::Print() << "iter = " << iter << "\n"; ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, geom, alpha_cc); average_cc_to_nodes(Nodal_PoissonRHS, PoissonRHS, geom); average_cc_to_nodes(alpha_nd, alpha_cc, geom); - Nodal_PoissonRHS.OverrideSync(geom.periodicity()); - alpha_nd.OverrideSync(geom.periodicity()); - #ifdef AMREX_USE_EB p_mlebabec->setACoeffs(0, alpha_cc); #else @@ -406,6 +402,7 @@ void main_main (c_FerroX& rFerroX) amrex::Print() << iter << " iterations :: err = " << err << std::endl; } + } amrex::Print() << "\n ========= Self-Consistent Initialization of P and Rho Done! ========== \n"<< iter << " iterations to obtain self consistent Phi with err = " << err << std::endl; // Calculate E from Phi @@ -416,8 +413,7 @@ void main_main (c_FerroX& rFerroX) // Write a plotfile of the initial data if plot_int > 0 if (plot_int > 0) { - //int step = 0; - int step = iter; + int step = 0; const std::string& pltfile = amrex::Concatenate("plt",step,8); MultiFab::Copy(Plt, P_old[0], 0, 0, 1, 0); MultiFab::Copy(Plt, P_old[1], 0, 1, 1, 0); @@ -446,11 +442,14 @@ void main_main (c_FerroX& rFerroX) #endif } - } amrex::Print() << "\n ========= Advance Steps ========== \n"<< std::endl; int steady_state_step = 1000000; //Initialize to a large number. It will be overwritten by the time step at which steady state condition is satidfied + int sign = 1; //change sign to -1*sign whenever abs(Phi_Bc_hi) == Phi_Bc_hi_max to do triangular wave sweep + int num_Vapp = 0; + Real tiny = 1.e-6; + for (int step = 1; step <= nsteps; ++step) { Real step_strt_time = ParallelDescriptor::second(); @@ -485,64 +484,53 @@ void main_main (c_FerroX& rFerroX) // iterate to compute Phi^{n+1,*} while(err > tol){ - //Compute x_H - SetPhiBC_z(Nodal_PoissonPhi_BC, n_cell); - - //Compute A x_H - pMLMG->apply ({&APoissonPhi_BC}, {&Nodal_PoissonPhi_BC}); - - //Compute b - ComputePoissonRHS(PoissonRHS, P_new_pre, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + ComputePoissonRHS(PoissonRHS, P_new_pre, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); - average_cc_to_nodes(Nodal_PoissonRHS, PoissonRHS, geom); + dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_new_pre, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); - //Compute b - A x_H - MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); + ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, geom, alpha_cc); - //dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_new_pre, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); + average_cc_to_nodes(Nodal_PoissonRHS, PoissonRHS, geom); - //ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); + average_cc_to_nodes(alpha_nd, alpha_cc, geom); -//#ifdef AMREX_USE_EB -// p_mlebabec->setACoeffs(0, alpha_cc); -//#else -// p_mlabec->setACoeffs(0, alpha_cc); -//#endif - //Initial guess for phi - Nodal_PoissonPhi.setVal(0.); - - //Poisson Solve - pMLMG->solve({&Nodal_PoissonPhi}, {&Nodal_PoissonRHS}, 1.e-10, -1); - - //Fill inhomogenous Dirichlet BC in boundary nodes after the solve - SetPhiBC_z_after_solve(Nodal_PoissonPhi, n_cell); - - Nodal_PoissonPhi.FillBoundary(geom.periodicity()); - - // Calculate rho from Phi in SC region - //ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); - //Nodal - amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); - ComputeRho(PoissonPhi, charge_den, e_den, hole_den, geom, MaterialMask); - - if (contains_SC == 0) { - // no semiconductor region; set error to zero so the while loop terminates - err = 0.; - } else { - - // Calculate Error - if (iter > 0){ - MultiFab::Copy(Nodal_PhiErr, Nodal_PoissonPhi, 0, 0, 1, 1); - MultiFab::Subtract(Nodal_PhiErr, Nodal_PoissonPhi_Prev, 0, 0, 1, 1); - err = Nodal_PhiErr.norm1(0, geom.periodicity())/Nodal_PoissonPhi.norm1(0, geom.periodicity()); - } - - //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration - MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 1); - - iter = iter + 1; - amrex::Print() << iter << " iterations :: err = " << err << std::endl; - } +#ifdef AMREX_USE_EB + p_mlebabec->setACoeffs(0, alpha_cc); +#else + p_mlndabec->setACoeffs(0, alpha_nd); +#endif + + //Initial guess for phi + Nodal_PoissonPhi.setVal(0.); + SetPhiBC_z(Nodal_PoissonPhi, n_cell); + + //Poisson Solve + pMLMG->solve({&Nodal_PoissonPhi}, {&Nodal_PoissonRHS}, 1.e-10, -1); + + Nodal_PoissonPhi.FillBoundary(geom.periodicity()); + + amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); + + // Calculate rho from Phi in SC region + ComputeRho(PoissonPhi, charge_den, e_den, hole_den, geom, MaterialMask); + + if (contains_SC == 0) { + // no semiconductor region; set error to zero so the while loop terminates + err = 0.; + } else { + + // Calculate Error + if (iter > 0){ + MultiFab::Copy(Nodal_PhiErr, Nodal_PoissonPhi, 0, 0, 1, 1); + MultiFab::Subtract(Nodal_PhiErr, Nodal_PoissonPhi_Prev, 0, 0, 1, 1); + err = Nodal_PhiErr.norm1(0, geom.periodicity())/Nodal_PoissonPhi.norm1(0, geom.periodicity()); + } + + MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 1); + + iter = iter + 1; + amrex::Print() << iter << " iterations :: err = " << err << std::endl; + } } if (TimeIntegratorOrder == 1) { @@ -572,65 +560,53 @@ void main_main (c_FerroX& rFerroX) // iterate to compute Phi^{n+1} while(err > tol){ - //Compute x_H - SetPhiBC_z(Nodal_PoissonPhi_BC, n_cell); - - //Compute A x_H - pMLMG->apply ({&APoissonPhi_BC}, {&Nodal_PoissonPhi_BC}); - - //Compute b - ComputePoissonRHS(PoissonRHS, P_new, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + ComputePoissonRHS(PoissonRHS, P_new, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); - average_cc_to_nodes(Nodal_PoissonRHS, PoissonRHS, geom); + dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_new, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); - //Compute b - A x_H - MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); + ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, geom, alpha_cc); - //dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_new, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); + average_cc_to_nodes(Nodal_PoissonRHS, PoissonRHS, geom); - //ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); - -//#ifdef AMREX_USE_EB -// p_mlebabec->setACoeffs(0, alpha_cc); -//#else -// p_mlabec->setACoeffs(0, alpha_cc); -//#endif - - //Initial guess for phi - Nodal_PoissonPhi.setVal(0.); + average_cc_to_nodes(alpha_nd, alpha_cc, geom); - //Poisson Solve - pMLMG->solve({&Nodal_PoissonPhi}, {&Nodal_PoissonRHS}, 1.e-10, -1); - - //Fill inhomogenous Dirichlet BC in boundary nodes after the solve - SetPhiBC_z_after_solve(Nodal_PoissonPhi, n_cell); - - Nodal_PoissonPhi.FillBoundary(geom.periodicity()); - - // Calculate rho from Phi in SC region - //ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); - //Nodal - amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); - ComputeRho(PoissonPhi, charge_den, e_den, hole_den, geom, MaterialMask); - - if (contains_SC == 0) { - // no semiconductor region; set error to zero so the while loop terminates - err = 0.; - } else { - - // Calculate Error - if (iter > 0){ - MultiFab::Copy(Nodal_PhiErr, Nodal_PoissonPhi, 0, 0, 1, 1); - MultiFab::Subtract(Nodal_PhiErr, Nodal_PoissonPhi_Prev, 0, 0, 1, 1); - err = Nodal_PhiErr.norm1(0, geom.periodicity())/Nodal_PoissonPhi.norm1(0, geom.periodicity()); - } - - //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration - MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 1); - - iter = iter + 1; - amrex::Print() << iter << " iterations :: err = " << err << std::endl; - } +#ifdef AMREX_USE_EB + p_mlebabec->setACoeffs(0, alpha_cc); +#else + p_mlndabec->setACoeffs(0, alpha_nd); +#endif + + //Initial guess for phi + Nodal_PoissonPhi.setVal(0.); + SetPhiBC_z(Nodal_PoissonPhi, n_cell); + + //Poisson Solve + pMLMG->solve({&Nodal_PoissonPhi}, {&Nodal_PoissonRHS}, 1.e-10, -1); + + Nodal_PoissonPhi.FillBoundary(geom.periodicity()); + + amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); + + // Calculate rho from Phi in SC region + ComputeRho(PoissonPhi, charge_den, e_den, hole_den, geom, MaterialMask); + + if (contains_SC == 0) { + // no semiconductor region; set error to zero so the while loop terminates + err = 0.; + } else { + + // Calculate Error + if (iter > 0){ + MultiFab::Copy(Nodal_PhiErr, Nodal_PoissonPhi, 0, 0, 1, 1); + MultiFab::Subtract(Nodal_PhiErr, Nodal_PoissonPhi_Prev, 0, 0, 1, 1); + err = Nodal_PhiErr.norm1(0, geom.periodicity())/Nodal_PoissonPhi.norm1(0, geom.periodicity()); + } + + MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 1); + + iter = iter + 1; + amrex::Print() << iter << " iterations :: err = " << err << std::endl; + } } // copy new solution into old solution @@ -715,10 +691,11 @@ void main_main (c_FerroX& rFerroX) MultiFab::Copy(Plt, angle_beta, 0, 15, 1, 0); MultiFab::Copy(Plt, angle_theta, 0, 16, 1, 0); MultiFab::Copy(Plt, Phidiff, 0, 17, 1, 0); + MultiFab::Copy(Plt, alpha_cc, 0, 18, 1, 0); #ifdef AMREX_USE_EB - amrex::EB_WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "tphase","alpha", "beta", "theta", "PhiDiff"}, geom, time, step); + amrex::EB_WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "tphase","alpha", "beta", "theta", "PhiDiff", "alpha_cc"}, geom, time, step); #else - amrex::WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "tphase","alpha", "beta", "theta", "PhiDiff"}, geom, time, step); + amrex::WriteSingleLevelPlotfile(pltfile, Plt, {"Px","Py","Pz","Phi","PoissonRHS","Ex","Ey","Ez","holes","electrons","charge","epsilon", "mask", "tphase","alpha", "beta", "theta", "PhiDiff", "alpha_cc"}, geom, time, step); #endif } @@ -726,20 +703,13 @@ void main_main (c_FerroX& rFerroX) { //Update time-dependent Boundary Condition of Poisson's equation - amrex::Print() << "Applied voltage updated at time " << time << ", step = " << step << "\n"; - - Phi_Bc_hi += Phi_Bc_inc; - amrex::Print() << "step = " << step << ", Phi_Bc_hi = " << Phi_Bc_hi << std::endl; - - // Set Dirichlet BC for Phi in z - //SetPhiBC_z(Nodal_PoissonPhi, n_cell); - - // set Dirichlet BC by reading in the ghost cell values -//#ifdef AMREX_USE_EB -// p_mlebabec->setLevelBC(amrlev, &PoissonPhi); -//#else -// p_mlabec->setLevelBC(amrlev, &PoissonPhi); -//#endif + Phi_Bc_hi += sign*Phi_Bc_inc; + num_Vapp += 1; + if(std::abs(std::abs(Phi_Bc_hi) - Phi_Bc_hi_max) <= tiny) { + sign *= -1; + amrex::Print() << "Direction of voltage sweep is reversed. Phi_Bc_hi = " << Phi_Bc_hi << ", and Phi_Bc_hi_max = " << Phi_Bc_hi_max << std::endl; + } + amrex::Print() << "step = " << step << ", Phi_Bc_hi = " << Phi_Bc_hi << ", num_Vapp = " << num_Vapp << ", sign = " << sign << std::endl; err = 1.0; iter = 0; @@ -747,72 +717,74 @@ void main_main (c_FerroX& rFerroX) // iterate to compute Phi^{n+1} with new Dirichlet value while(err > tol){ - //Compute x_H - SetPhiBC_z(Nodal_PoissonPhi_BC, n_cell); - - //Compute A x_H - pMLMG->apply ({&APoissonPhi_BC}, {&Nodal_PoissonPhi_BC}); - - //Compute b - ComputePoissonRHS(PoissonRHS, P_old, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); + ComputePoissonRHS(PoissonRHS, P_old, charge_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom); - average_cc_to_nodes(Nodal_PoissonRHS, PoissonRHS, geom); + dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); - //Compute b - A x_H - MultiFab::Subtract(Nodal_PoissonRHS, APoissonPhi_BC, 0, 0, 1, 0); + ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, geom, alpha_cc); - //dF_dPhi(alpha_cc, PoissonRHS, PoissonPhi, P_old, charge_den, e_den, hole_den, MaterialMask, angle_alpha, angle_beta, angle_theta, geom, prob_lo, prob_hi); + average_cc_to_nodes(Nodal_PoissonRHS, PoissonRHS, geom); - //ComputePoissonRHS_Newton(PoissonRHS, PoissonPhi, alpha_cc); + average_cc_to_nodes(alpha_nd, alpha_cc, geom); -//#ifdef AMREX_USE_EB -// p_mlebabec->setACoeffs(0, alpha_cc); -//#else -// p_mlabec->setACoeffs(0, alpha_cc); -//#endif - - //Initial guess for phi - Nodal_PoissonPhi.setVal(0.); - - //Poisson Solve - pMLMG->solve({&Nodal_PoissonPhi}, {&Nodal_PoissonRHS}, 1.e-10, -1); - - //Fill inhomogenous Dirichlet BC in boundary nodes after the solve - SetPhiBC_z_after_solve(Nodal_PoissonPhi, n_cell); - - Nodal_PoissonPhi.FillBoundary(geom.periodicity()); - - // Calculate rho from Phi in SC region - //ComputeRho(PoissonPhi, charge_den, e_den, hole_den, MaterialMask); - //Nodal - amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); - ComputeRho(PoissonPhi, charge_den, e_den, hole_den, geom, MaterialMask); - - if (contains_SC == 0) { - // no semiconductor region; set error to zero so the while loop terminates - err = 0.; - } else { - - // Calculate Error - if (iter > 0){ - MultiFab::Copy(Nodal_PhiErr, Nodal_PoissonPhi, 0, 0, 1, 1); - MultiFab::Subtract(Nodal_PhiErr, Nodal_PoissonPhi_Prev, 0, 0, 1, 1); - err = Nodal_PhiErr.norm1(0, geom.periodicity())/Nodal_PoissonPhi.norm1(0, geom.periodicity()); - } - - //Copy PoissonPhi to PoissonPhi_Prev to calculate error at the next iteration - MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 1); - - iter = iter + 1; - amrex::Print() << iter << " iterations :: err = " << err << std::endl; - } +#ifdef AMREX_USE_EB + p_mlebabec->setACoeffs(0, alpha_cc); +#else + p_mlndabec->setACoeffs(0, alpha_nd); +#endif + + //Initial guess for phi + Nodal_PoissonPhi.setVal(0.); + SetPhiBC_z(Nodal_PoissonPhi, n_cell); + + //Poisson Solve + pMLMG->solve({&Nodal_PoissonPhi}, {&Nodal_PoissonRHS}, 1.e-10, -1); + + Nodal_PoissonPhi.FillBoundary(geom.periodicity()); + + amrex::average_node_to_cellcenter(PoissonPhi, 0, Nodal_PoissonPhi, 0, 1); + + // Calculate rho from Phi in SC region + ComputeRho(PoissonPhi, charge_den, e_den, hole_den, geom, MaterialMask); + + if (contains_SC == 0) { + // no semiconductor region; set error to zero so the while loop terminates + err = 0.; + } else { + + // Calculate Error + if (iter > 0){ + MultiFab::Copy(Nodal_PhiErr, Nodal_PoissonPhi, 0, 0, 1, 1); + MultiFab::Subtract(Nodal_PhiErr, Nodal_PoissonPhi_Prev, 0, 0, 1, 1); + err = Nodal_PhiErr.norm1(0, geom.periodicity())/Nodal_PoissonPhi.norm1(0, geom.periodicity()); + } + + MultiFab::Copy(Nodal_PoissonPhi_Prev, Nodal_PoissonPhi, 0, 0, 1, 1); + + iter = iter + 1; + amrex::Print() << iter << " iterations :: err = " << err << std::endl; + } } }//end inc_step - - if (voltage_sweep == 0 && step == steady_state_step) break; - if (voltage_sweep == 1 && Phi_Bc_hi > Phi_Bc_hi_max) break; - + + if (voltage_sweep == 0 && step == steady_state_step) { + amrex::Print() << "voltage_sweep == 0 && step == steady_state_step!" << "\n"; + break; + } + if (voltage_sweep == 1 && Phi_Bc_hi > 0. && Phi_Bc_hi - Phi_Bc_hi_max > tiny) { + amrex::Print() << "voltage_sweep == 1 && Phi_Bc_hi > 0. && Phi_Bc_hi - Phi_Bc_hi_max > tiny!" << "\n"; + break; + } + if (voltage_sweep == 1 && Phi_Bc_hi < 0. && -Phi_Bc_hi - Phi_Bc_hi_max > tiny) { + amrex::Print() << "voltage_sweep == 1 && Phi_Bc_hi < 0. && -Phi_Bc_hi - Phi_Bc_hi_max > tiny!" << "\n"; + break; + } + if (voltage_sweep == 1 && num_Vapp == num_Vapp_max) { + amrex::Print() << "voltage_sweep == 1 && num_Vapp == num_Vapp_max!" << "\n"; + break; + } + } // end step // MultiFab memory usage